Skip to content

Commit

Permalink
added regression flag to exit 1 in event of regression
Browse files Browse the repository at this point in the history
Signed-off-by: Shashank Reddy Boyapally <[email protected]>
  • Loading branch information
shashank-boyapally committed Aug 7, 2024
1 parent 08cef10 commit 72ad366
Show file tree
Hide file tree
Showing 7 changed files with 24 additions and 15 deletions.
4 changes: 3 additions & 1 deletion orion.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ def cmd_analysis(**kwargs):
logger_instance = SingletonLogger(debug=level, name="Orion")
logger_instance.info("🏹 Starting Orion in command-line mode")
kwargs["configMap"] = load_config(kwargs["config"])
output = run(**kwargs)
output, regression_flag = run(**kwargs)
if output is None:
logger_instance.error("Terminating test")
sys.exit(0)
Expand All @@ -125,6 +125,8 @@ def cmd_analysis(**kwargs):
output_file_name = f"{kwargs['save_output_path'].split('.')[0]}_{test_name}.{kwargs['save_output_path'].split('.')[1]}"
with open(output_file_name, 'w', encoding="utf-8") as file:
file.write(str(result_table))
if regression_flag:
sys.exit(1)



Expand Down
13 changes: 7 additions & 6 deletions pkg/algorithms/algorithm.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,9 @@ def __init__( # pylint: disable = too-many-arguments
self.test = test
self.options = options
self.metrics_config = metrics_config
self.regression_flag = False

def output_json(self) -> Tuple[str, str]:
def output_json(self) -> Tuple[str, str, bool]:
"""Method to output json output
Returns:
Expand Down Expand Up @@ -64,9 +65,9 @@ def output_json(self) -> Tuple[str, str]:
] = percentage_change
dataframe_json[index]["is_changepoint"] = True

return self.test["name"], json.dumps(dataframe_json, indent=2)
return self.test["name"], json.dumps(dataframe_json, indent=2), self.regression_flag

def output_text(self) -> Tuple[str,str]:
def output_text(self) -> Tuple[str,str, bool]:
"""Outputs the data in text/tabular format"""
series, change_points_by_metric = self._analyze()
change_points_by_time = self.group_change_points_by_time(
Expand All @@ -76,9 +77,9 @@ def output_text(self) -> Tuple[str,str]:
output_table = report.produce_report(
test_name=self.test["name"], report_type=ReportType.LOG
)
return self.test["name"], output_table
return self.test["name"], output_table, self.regression_flag

def output_junit(self) -> Tuple[str,str]:
def output_junit(self) -> Tuple[str,str, bool]:
"""Output junit format
Returns:
Expand All @@ -92,7 +93,7 @@ def output_junit(self) -> Tuple[str,str]:
metrics_config=self.metrics_config,
options=self.options,
)
return test_name, data_junit
return test_name, data_junit, self.regression_flag

@abstractmethod
def _analyze(self):
Expand Down
3 changes: 2 additions & 1 deletion pkg/algorithms/edivisive/edivisive.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,5 +25,6 @@ def _analyze(self):
if ((self.metrics_config[metric]["direction"] == 1 and changepoint_list[i].stats.mean_1 > changepoint_list[i].stats.mean_2) or
(self.metrics_config[metric]["direction"] == -1 and changepoint_list[i].stats.mean_1 < changepoint_list[i].stats.mean_2) ):
del changepoint_list[i]

if [val for li in change_points_by_metric.values() for val in li]:
self.regression_flag=True
return series, change_points_by_metric
3 changes: 2 additions & 1 deletion pkg/algorithms/isolationforest/isolationForest.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,5 +70,6 @@ def _analyze(self):
pvalue=1
))
change_points_by_metric[feature].append(change_point)

if [val for li in change_points_by_metric.values() for val in li]:
self.regression_flag=True
return series, change_points_by_metric
6 changes: 4 additions & 2 deletions pkg/daemon.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,8 @@ async def daemon_changepoint( # pylint: disable = R0913
filter_changepoints = (
True if filter_changepoints == "true" else False # pylint: disable = R1719
)
result = runTest.run(**option_arguments)
result, _ = runTest.run(**option_arguments)
result = {k:json.loads(v) for k,v in result.items()}
if result is None:
return {"Error":"No UUID with given metadata"}
result = {k:json.loads(v) for k,v in result.items()}
Expand Down Expand Up @@ -129,7 +130,8 @@ async def daemon_anomaly( # pylint: disable = R0913, R0914
filter_points = (
True if filter_points == "true" else False # pylint: disable = R1719
)
result = runTest.run(**option_arguments)
result, _ = runTest.run(**option_arguments)
result = {k:json.loads(v) for k,v in result.items()}
if result is None:
return {"Error":"No UUID with given metadata"}
result = {k:json.loads(v) for k,v in result.items()}
Expand Down
9 changes: 5 additions & 4 deletions pkg/runTest.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@



def run(**kwargs: dict[str, Any]) -> dict[str, Any]:
def run(**kwargs: dict[str, Any]) -> dict[str, Any]: #pylint: disable = R0914
"""run method to start the tests
Args:
Expand All @@ -26,6 +26,7 @@ def run(**kwargs: dict[str, Any]) -> dict[str, Any]:
config_map = kwargs["configMap"]
datasource = get_datasource(config_map)
result_output = {}
regression_flag = False
for test in config_map["tests"]:
# Create fingerprint Matcher
matcher = Matcher(
Expand All @@ -42,7 +43,6 @@ def run(**kwargs: dict[str, Any]) -> dict[str, Any]:
kwargs,
start_timestamp,
)

if fingerprint_matched_df is None:
return None

Expand All @@ -62,9 +62,10 @@ def run(**kwargs: dict[str, Any]) -> dict[str, Any]:
kwargs,
metrics_config,
)
testname, result_data = algorithm.output(kwargs["output_format"])
testname, result_data, test_flag = algorithm.output(kwargs["output_format"])
result_output[testname] = result_data
return result_output
regression_flag = regression_flag or test_flag
return result_output, regression_flag


def get_start_timestamp(kwargs: Dict[str, Any]) -> str:
Expand Down
1 change: 1 addition & 0 deletions pkg/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,7 @@ def process_test(
else buildUrls[uuid]
) # pylint: disable = cell-var-from-loop
)
merged_df=merged_df.reset_index(drop=True)
#save the dataframe
output_file_path = f"{options['save_data_path'].split('.')[0]}-{test['name']}.csv"
match.save_results(merged_df, csv_file_path=output_file_path)
Expand Down

0 comments on commit 72ad366

Please sign in to comment.