From 22a8614ec87f9e8c1c94cb1c77a51cef16963a13 Mon Sep 17 00:00:00 2001 From: sousinha1997 Date: Tue, 27 Aug 2024 17:50:29 +0530 Subject: [PATCH] push --- quisby/benchmarks/auto_hpl/extract.py | 12 +- quisby/benchmarks/coremark/compare.py | 20 ++- quisby/benchmarks/coremark/coremark.py | 113 ++++++++---- quisby/benchmarks/coremark/graph.py | 4 +- .../benchmarks/coremark_pro/coremark_pro.py | 114 ++++++++---- quisby/benchmarks/coremark_pro/graph.py | 4 +- quisby/benchmarks/linpack/comparison.py | 2 +- quisby/benchmarks/linpack/extract.py | 19 +- quisby/benchmarks/linpack/graph.py | 8 +- quisby/benchmarks/linpack/summary.py | 46 ++++- quisby/benchmarks/passmark/compare.py | 7 +- quisby/benchmarks/passmark/graph.py | 4 +- quisby/benchmarks/passmark/passmark.py | 156 +++++++++-------- quisby/benchmarks/phoronix/compare.py | 20 ++- quisby/benchmarks/phoronix/graph.py | 4 +- quisby/benchmarks/phoronix/phoronix.py | 141 ++++++++------- quisby/benchmarks/pyperf/compare.py | 13 +- quisby/benchmarks/pyperf/graph.py | 4 +- quisby/benchmarks/pyperf/pyperf.py | 165 +++++++++--------- quisby/benchmarks/specjbb/comparison.py | 5 +- quisby/benchmarks/specjbb/graph.py | 46 ++++- quisby/benchmarks/specjbb/specjbb.py | 50 ++++-- 22 files changed, 584 insertions(+), 373 deletions(-) diff --git a/quisby/benchmarks/auto_hpl/extract.py b/quisby/benchmarks/auto_hpl/extract.py index bd922a8..1a0a3b4 100644 --- a/quisby/benchmarks/auto_hpl/extract.py +++ b/quisby/benchmarks/auto_hpl/extract.py @@ -3,13 +3,21 @@ from quisby.pricing import cloud_pricing from quisby.benchmarks.linpack.extract import linpack_format_data +from quisby.util import read_config + def extract_auto_hpl_data(path, system_name): + summary_data = [] + server = read_config("server", "name") + result_dir = read_config("server", "result_dir") + if path.endswith(".csv"): with open(path) as file: results = [] file_data = file.readlines() + sum_path = path.split("/./")[1] + summary_data.append([system_name, "http://" + server + "/results/" + result_dir + "/" + sum_path]) if len(file_data) > 1: header_row = file_data[-2].strip().split(":") @@ -24,8 +32,8 @@ def extract_auto_hpl_data(path, system_name): ) if results: - return results + return results, summary_data else: - return None + return None, None diff --git a/quisby/benchmarks/coremark/compare.py b/quisby/benchmarks/coremark/compare.py index fdea253..aa07ce8 100644 --- a/quisby/benchmarks/coremark/compare.py +++ b/quisby/benchmarks/coremark/compare.py @@ -17,8 +17,9 @@ def extract_prefix_and_number(input_string): match = re.search(r'^(.*?)(\d+)(.*?)$', input_string) if match: prefix = match.group(1) - return prefix - return None + suffix = match.group(3) # Extracts the suffix after the number + return prefix, suffix + return None, None def compare_inst(item1, item2): @@ -33,7 +34,7 @@ def compare_inst(item1, item2): return extract_prefix_and_number(item1) == extract_prefix_and_number(item2) -def compare_coremark_results(spreadsheets, spreadsheetId, test_name, table_name=["System name","Price/perf"]): +def compare_coremark_results(spreadsheets, spreadsheetId, test_name, table_name=["System name","Price-perf"]): values = [] results = [] spreadsheet_name = [] @@ -51,12 +52,13 @@ def compare_coremark_results(spreadsheets, spreadsheetId, test_name, table_name= for ele in list_2: # Check max throughput if value[0][0] in table_name and ele[0][0] in table_name and value[0][0] == ele[0][0]: - results.append([""]) - for item1 in value: - for item2 in ele: - if item1[0] == item2[0]: - results = merge_lists_alternately(results, item1, item2) - break + if compare_inst(value[1][0], ele[1][0]): + results.append([""]) + for item1 in value: + for item2 in ele: + if item1[0] == item2[0]: + results = merge_lists_alternately(results, item1, item2) + break elif value[0][0] == "Cost/Hr" and ele[0][0] == "Cost/Hr": if compare_inst(value[1][0], ele[1][0]): diff --git a/quisby/benchmarks/coremark/coremark.py b/quisby/benchmarks/coremark/coremark.py index 4e5665b..9465a14 100644 --- a/quisby/benchmarks/coremark/coremark.py +++ b/quisby/benchmarks/coremark/coremark.py @@ -1,11 +1,17 @@ """ Custom key to sort the data base don instance name """ +from itertools import groupby + from quisby import custom_logger import re from quisby.util import read_config from quisby.pricing.cloud_pricing import get_cloud_pricing +from quisby.util import process_instance + +from quisby.util import mk_int + def extract_prefix_and_number(input_string): match = re.search(r'^(.*?)(\d+)(.*?)$', input_string) @@ -18,20 +24,21 @@ def extract_prefix_and_number(input_string): def custom_key(item): - cloud_type = read_config("cloud","cloud_type") + cloud_type = read_config("cloud", "cloud_type") if item[1][0] == "localhost": return item[1][0] elif cloud_type == "aws": - instance_type =item[1][0].split(".")[0] - instance_number = item[1][0].split(".")[1] + instance_name = item[1][0] + instance_type = instance_name.split(".")[0] + instance_number = instance_name.split(".")[1] return instance_type, instance_number elif cloud_type == "gcp": instance_type = item[1][0].split("-")[0] instance_number = int(item[1][0].split('-')[-1]) return instance_type, instance_number elif cloud_type == "azure": - instance_type, instance_number, version=extract_prefix_and_number(item[1][0]) - return instance_type, instance_number + instance_type, instance_number, version = extract_prefix_and_number(item[1][0]) + return instance_type, version, instance_number def calc_price_performance(inst, avg): @@ -39,6 +46,7 @@ def calc_price_performance(inst, avg): cloud_type = read_config("cloud", "cloud_type") os_type = read_config("test", "os_type") cost_per_hour = None + price_perf = 0.0 try: cost_per_hour = get_cloud_pricing( inst, region, cloud_type.lower(), os_type) @@ -49,40 +57,66 @@ def calc_price_performance(inst, avg): return cost_per_hour, price_perf -def create_summary_coremark_data(results, OS_RELEASE): +def group_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature", "machine_type")) + elif cloud_type == "azure": + results = sorted(results, key=lambda x: process_instance(x[1][0], "family", "feature")) + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "feature")) + elif cloud_type == "gcp": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version","sub_family","feature")) + + +def sort_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + results.sort(key=lambda x: str(process_instance(x[1][0], "family"))) + elif cloud_type == "azure": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "feature"))) + elif cloud_type == "gcp": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "sub_family"))) + +def create_summary_coremark_data(results, OS_RELEASE, sorted_results=None): final_results = [] - cal_data = [["System name", "Test_passes-"+OS_RELEASE]] # Sort data based on instance name - sorted_data = sorted(results, key=custom_key) - cost_per_hour, price_per_perf = [], [] - - # Add summary data - for item in sorted_data: - sum = 0 - avg = 0 - iterations = 0 - for index in range(3, len(item)): - sum = sum + float(item[index][1]) - iterations = iterations + 1 - avg = float(sum/iterations) - try: - cph, pp = calc_price_performance(item[1][0], avg) - except Exception as exc: - custom_logger.error(str(exc)) - break - cal_data.append([item[1][0], avg]) - price_per_perf.append([item[1][0], pp]) - cost_per_hour.append([item[1][0], cph]) - - final_results += [[""]] - final_results += cal_data - final_results.append([""]) - final_results.append(["Cost/Hr"]) - final_results += cost_per_hour - final_results.append([""]) - final_results.append(["Price/perf", f"Passes/$-{OS_RELEASE}"]) - final_results += price_per_perf + results = list(filter(None, results)) + sort_data(results) + + for _, items in group_data(results): + cal_data = [["System name", "Test_passes-" + OS_RELEASE]] + items = list(items) + sorted_data = sorted(items, key=lambda x: mk_int(process_instance(x[1][0], "size"))) + # sorted_results.extend(sorted_data) + cost_per_hour, price_per_perf = [], [] + + # Add summary data + for item in sorted_data: + sum = 0 + avg = 0 + iterations = 0 + for index in range(3, len(item)): + sum = sum + float(item[index][1]) + iterations = iterations + 1 + avg = float(sum/iterations) + try: + cph, pp = calc_price_performance(item[1][0], avg) + except Exception as exc: + custom_logger.error(str(exc)) + break + cal_data.append([item[1][0], avg]) + price_per_perf.append([item[1][0], pp]) + cost_per_hour.append([item[1][0], cph]) + sorted_results=[[""]] + sorted_results += cal_data + sorted_results.append([""]) + sorted_results.append(["Cost/Hr"]) + sorted_results += cost_per_hour + sorted_results.append([""]) + sorted_results.append(["Price-perf", f"Passes/$-{OS_RELEASE}"]) + sorted_results += price_per_perf + final_results.extend(sorted_results) return final_results @@ -90,12 +124,17 @@ def extract_coremark_data(path, system_name, OS_RELEASE): """""" results = [] processed_data =[] + summary_data = [] + server = read_config("server", "name") + result_dir = read_config("server", "result_dir") # Extract data from file try: if path.endswith(".csv"): with open(path) as file: coremark_results = file.readlines() + sum_path = path.split("/./")[1] + summary_data.append([system_name, "http://" + server + "/results/" + result_dir + "/" + sum_path]) else: return None except Exception as exc: @@ -118,7 +157,7 @@ def extract_coremark_data(path, system_name, OS_RELEASE): iteration = iteration + 1 results.append(processed_data) - return results + return results, summary_data diff --git a/quisby/benchmarks/coremark/graph.py b/quisby/benchmarks/coremark/graph.py index 01ac34b..3880cc3 100644 --- a/quisby/benchmarks/coremark/graph.py +++ b/quisby/benchmarks/coremark/graph.py @@ -108,7 +108,7 @@ def graph_coremark_data(spreadsheetId, range, action): header_row.extend(row) title = "%s : %s" % (range, "Test Passes") subtitle = "Average Test Passes" - elif "Price/perf" in row: + elif "Price-perf" in row: start_index = index header_row.extend(row) title = "%s : %s" % (range, "Price-Performance") @@ -133,7 +133,7 @@ def graph_coremark_data(spreadsheetId, range, action): "chart": { "spec": { "title": title, - "subtitle": subtitle, + "subtitle": subtitle + " : "+graph_data[1][0], "basicChart": { "chartType": "COMBO", "legendPosition": "RIGHT_LEGEND", diff --git a/quisby/benchmarks/coremark_pro/coremark_pro.py b/quisby/benchmarks/coremark_pro/coremark_pro.py index 4a11362..25fa0c1 100644 --- a/quisby/benchmarks/coremark_pro/coremark_pro.py +++ b/quisby/benchmarks/coremark_pro/coremark_pro.py @@ -1,9 +1,12 @@ import re +from itertools import groupby from quisby import custom_logger from quisby.util import read_config from quisby.pricing.cloud_pricing import get_cloud_pricing +from quisby.util import process_instance, mk_int + def extract_prefix_and_number(input_string): match = re.search(r'^(.*?)(\d+)(.*?)$', input_string) @@ -30,7 +33,7 @@ def custom_key(item): return instance_type, instance_number elif cloud_type == "azure": instance_type, instance_number, version = extract_prefix_and_number(item[1][0]) - return instance_type, instance_number + return instance_type, version, instance_number except Exception as exc: custom_logger.error(str(exc)) return "", "" @@ -41,6 +44,7 @@ def calc_price_performance(inst, avg): cloud_type = read_config("cloud", "cloud_type") os_type = read_config("test", "os_type") cost_per_hour = None + price_perf = 0.0 try: cost_per_hour = get_cloud_pricing( inst, region, cloud_type.lower(), os_type) @@ -51,60 +55,94 @@ def calc_price_performance(inst, avg): return cost_per_hour, price_perf +def group_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature", "machine_type")) + elif cloud_type == "azure": + results = sorted(results, key=lambda x: process_instance(x[1][0], "family", "feature")) + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature")) + elif cloud_type == "gcp": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version","sub_family","feature")) + + +def sort_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + results.sort(key=lambda x: str(process_instance(x[1][0], "family"))) + elif cloud_type == "azure": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "feature"))) + elif cloud_type == "gcp": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "sub_family"))) + + def create_summary_coremark_pro_data(results, OS_RELEASE): - final_results = [] - multi_iter = [["Multi Iterations"], ["System name", "Score-" + OS_RELEASE]] - single_iter = [["Single Iterations"], ["System name", "Score-" + OS_RELEASE]] + ret_results = [] + # Sort data based on instance name - sorted_data = sorted(results, key=custom_key) - # Add summary data - cost_per_hour, price_perf_single, price_perf_multi = [], [],[] - for item in sorted_data: - for index in range(3, len(item)): - multi_iter.append([item[1][0], item[index][1]]) - single_iter.append([item[1][0], item[index][2]]) - try: - cph, ppm = calc_price_performance(item[1][0], item[index][1]) - cph, pps = calc_price_performance(item[1][0], item[index][2]) - except Exception as exc: - custom_logger.error(str(exc)) - break - price_perf_multi.append([item[1][0], ppm]) - price_perf_single.append([item[1][0], pps]) - cost_per_hour.append([item[1][0], cph]) - # final_results += item - final_results += [[""]] - final_results += single_iter - final_results.append([""]) - final_results.append(["Cost/Hr"]) - final_results += cost_per_hour - final_results.extend([[""], ["Single Iterations"]]) - final_results.append(["Price/perf", f"Score/$-{OS_RELEASE}"]) - final_results += price_perf_single - final_results += [[""]] - final_results += multi_iter - final_results.extend([[""], ["Multi Iterations"]]) - final_results.append(["Price/perf", f"Score/$-{OS_RELEASE}"]) - final_results += price_perf_multi - return final_results + results = list(filter(None, results)) + sort_data(results) + results = group_data(results) + for _, items in results: + multi_iter = [["Multi Iterations"], ["System name", "Score-" + OS_RELEASE]] + single_iter = [["Single Iterations"], ["System name", "Score-" + OS_RELEASE]] + cal_data = [["System name", "Test_passes-" + OS_RELEASE]] + items = list(items) + sorted_data = sorted(items, key=lambda x: mk_int(process_instance(x[1][0], "size"))) + # Add summary data + cost_per_hour, price_perf_single, price_perf_multi = [], [],[] + for item in sorted_data: + for index in range(3, len(item)): + multi_iter.append([item[1][0], item[index][1]]) + single_iter.append([item[1][0], item[index][2]]) + try: + cph, ppm = calc_price_performance(item[1][0], item[index][1]) + cph, pps = calc_price_performance(item[1][0], item[index][2]) + except Exception as exc: + custom_logger.error(str(exc)) + break + price_perf_multi.append([item[1][0], ppm]) + price_perf_single.append([item[1][0], pps]) + cost_per_hour.append([item[1][0], cph]) + # final_results += item + final_results =[[""]] + final_results += single_iter + final_results.append([""]) + final_results.append(["Cost/Hr"]) + final_results += cost_per_hour + final_results.extend([[""], ["Single Iterations"]]) + final_results.append(["Price-perf", f"Score/$-{OS_RELEASE}"]) + final_results += price_perf_single + final_results += [[""]] + final_results += multi_iter + final_results.extend([[""], ["Multi Iterations"]]) + final_results.append(["Price-perf", f"Score/$-{OS_RELEASE}"]) + final_results += price_perf_multi + ret_results.extend(final_results) + return ret_results def extract_coremark_pro_data(path, system_name, OS_RELEASE): """""" results = [] processed_data = [] + summary_data = [] + server = read_config("server", "name") + result_dir = read_config("server", "result_dir") # Extract data from file try: if path.endswith(".csv"): with open(path) as file: coremark_results = file.readlines() + sum_path = path.split("/./")[1] + summary_data.append([system_name, "http://" + server + "/results/" + result_dir + "/" + sum_path]) else: - return None + return None,None except Exception as exc: custom_logger.debug(str(exc)) custom_logger.error("Unable to extract data from csv file for coremark_pro") - return None + return None, None for index, data in enumerate(coremark_results): coremark_results[index] = data.strip("\n").split(":") @@ -119,4 +157,4 @@ def extract_coremark_pro_data(path, system_name, OS_RELEASE): elif "Score" in row: processed_data.append(["Score", row[1], row[2]]) results.append(processed_data) - return results \ No newline at end of file + return results, summary_data \ No newline at end of file diff --git a/quisby/benchmarks/coremark_pro/graph.py b/quisby/benchmarks/coremark_pro/graph.py index fac3af9..af9eebd 100644 --- a/quisby/benchmarks/coremark_pro/graph.py +++ b/quisby/benchmarks/coremark_pro/graph.py @@ -110,7 +110,7 @@ def graph_coremark_pro_data(spreadsheetId, range, action): iteration = data[index - 1][0] title = "%s : %s" % (range, "Score") subtitle = iteration - elif "Price/perf" in row: + elif "Price-perf" in row: start_index = index header.extend(row) iteration = data[index - 1][0] @@ -138,7 +138,7 @@ def graph_coremark_pro_data(spreadsheetId, range, action): "chart": { "spec": { "title": title, - "subtitle": subtitle, + "subtitle": subtitle+" : "+graph_data[1][0], "basicChart": { "chartType": "COMBO", "legendPosition": "RIGHT_LEGEND", diff --git a/quisby/benchmarks/linpack/comparison.py b/quisby/benchmarks/linpack/comparison.py index f2c5f5d..e28cb71 100644 --- a/quisby/benchmarks/linpack/comparison.py +++ b/quisby/benchmarks/linpack/comparison.py @@ -38,7 +38,7 @@ def compare_linpack_results(spreadsheets, spreadsheetId, test_name): value[4], value[5], ele[5], - "Price/Perf % Diff", + "Price-perf % Diff", ] ) break diff --git a/quisby/benchmarks/linpack/extract.py b/quisby/benchmarks/linpack/extract.py index 0b21c7d..ac7bd57 100644 --- a/quisby/benchmarks/linpack/extract.py +++ b/quisby/benchmarks/linpack/extract.py @@ -31,16 +31,6 @@ def linpack_format_data(**kwargs): system_name, region, cloud_type.lower() ) - results.append( - [ - "System", - "Cores", - f"GFLOPS-{os_release}", - f"GFLOP Scaling-{os_release}", - "Cost/hr", - f"Price/Perf-{os_release}", - ] - ) results.append( [ system_name, @@ -64,6 +54,9 @@ def extract_linpack_data(path, system_name): results = [] no_of_cores = None gflops = None + summary_data = [] + server = read_config("server", "name") + result_dir = read_config("server", "result_dir") summary_file = path @@ -77,8 +70,10 @@ def extract_linpack_data(path, system_name): last_row = list_data[-1] gflops = last_row["MB/sec"] threads = last_row["threads"] + sum_path = summary_file.split("/./")[1] + summary_data.append([system_name, "http://" + server + "/results/" + result_dir + "/" + sum_path]) else: - return results + return results, summary_data for file_path in glob.glob(path + f"/linpack*_threads_{threads}_*"): with open(file_path) as txt_file: @@ -96,4 +91,4 @@ def extract_linpack_data(path, system_name): gflops=gflops, ) - return results + return results, summary_data diff --git a/quisby/benchmarks/linpack/graph.py b/quisby/benchmarks/linpack/graph.py index 7b72483..e9b0d2d 100644 --- a/quisby/benchmarks/linpack/graph.py +++ b/quisby/benchmarks/linpack/graph.py @@ -45,7 +45,7 @@ def graph_linpack_compare(spreadsheetId, test_name, action): Graphs: - GFLOP and GFLOPS scaling - - Price/perf + - Price-perf :sheet: sheet API function :spreadsheetId @@ -203,7 +203,7 @@ def graph_linpack_compare(spreadsheetId, test_name, action): sheet.batchUpdate(spreadsheetId=spreadsheetId, body=body).execute() - # PRICE/PERF graph + # Price-perf graph requests = { "addChart": { "chart": { @@ -462,7 +462,7 @@ def graph_linpack_data(spreadsheetId, test_name, action): Graphs: - GFLOP and GFLOPS scaling - - Price/perf + - Price-perf :sheet: sheet API function :spreadsheetId @@ -602,7 +602,7 @@ def graph_linpack_data(spreadsheetId, test_name, action): } } - # PRICE/PERF graph + # Price-perf graph body = {"requests": requests} sheet.batchUpdate(spreadsheetId=spreadsheetId, body=body).execute() diff --git a/quisby/benchmarks/linpack/summary.py b/quisby/benchmarks/linpack/summary.py index 4da5fcb..6330cf2 100644 --- a/quisby/benchmarks/linpack/summary.py +++ b/quisby/benchmarks/linpack/summary.py @@ -28,19 +28,50 @@ def custom_key(item): return instance_type, instance_number elif cloud_type == "azure": instance_type, instance_number, version=extract_prefix_and_number(item[0]) - return instance_type, instance_number + return instance_type, version, instance_number + + +def group_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + return groupby(results, key=lambda x: process_instance(x[0], "family", "version", "feature", "machine_type")) + elif cloud_type == "azure": + results = sorted(results, key=lambda x: process_instance(x[0], "family", "feature")) + return groupby(results, key=lambda x: process_instance(x[0], "family", "version", "feature")) + elif cloud_type == "gcp": + return groupby(results, key=lambda x: process_instance(x[0], "family", "version","sub_family","feature")) + + + +def sort_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + results.sort(key=lambda x: str(process_instance(x[0], "family"))) + elif cloud_type == "azure": + results.sort(key=lambda x: str(process_instance(x[0], "family", "version", "feature"))) + elif cloud_type == "gcp": + results.sort(key=lambda x: str(process_instance(x[0], "family", "version", "sub_family"))) def create_summary_linpack_data(results, OS_RELEASE): sorted_results = [] + header = [] + header.append( + [ + "System", + "Cores", + f"GFLOPS-{OS_RELEASE}", + f"GFLOP Scaling-{OS_RELEASE}", + "Cost/hr", + f"Price-perf-{OS_RELEASE}", + ] + ) results = list(filter(None, results)) - header_row = [results[0]] - results = [row for row in results if row[0] != "System"] - - results.sort(key=lambda x: str((x[0], "family", "version","sub_family", "feature"))) + sort_data(results) + #results.sort(key=lambda x: str((x[0], "family", "version","sub_family", "feature"))) - for _, items in groupby(results, key=lambda x: process_instance(x[0], "family", "version","sub_family", "feature")): + for _, items in group_data(results): items = list(items) sorted_data = sorted(items, key=lambda x: mk_int(process_instance(x[0], "size"))) cpu_scale, base_gflops = None, None @@ -55,11 +86,10 @@ def create_summary_linpack_data(results, OS_RELEASE): cpu_scaling = 0 gflops_scaling = float(row[2]) / (int(row[1]) - cpu_scale) / base_gflops if cpu_scaling != 0 else 1 sorted_data[index][3] = format(gflops_scaling, ".4f") - sorted_data = sorted(sorted_data, key=custom_key) res = [] for item in sorted_data: res.append(item) - sorted_results += header_row + res + sorted_results += header + res # sorted_results += header_row + sorted_data return sorted_results diff --git a/quisby/benchmarks/passmark/compare.py b/quisby/benchmarks/passmark/compare.py index f41a8d0..5dc0c17 100644 --- a/quisby/benchmarks/passmark/compare.py +++ b/quisby/benchmarks/passmark/compare.py @@ -16,8 +16,9 @@ def extract_prefix_and_number(input_string): match = re.search(r'^(.*?)(\d+)(.*?)$', input_string) if match: prefix = match.group(1) - return prefix - return None + suffix = match.group(3) # Extracts the suffix after the number + return prefix, suffix + return None, None def compare_inst(item1, item2): @@ -32,7 +33,7 @@ def compare_inst(item1, item2): return extract_prefix_and_number(item1) == extract_prefix_and_number(item2) -def compare_passmark_results(spreadsheets, spreadsheetId, test_name, table_name=["System name","Price/perf"]): +def compare_passmark_results(spreadsheets, spreadsheetId, test_name, table_name=["System name","Price-perf"]): values = [] results = [] spreadsheet_name = [] diff --git a/quisby/benchmarks/passmark/graph.py b/quisby/benchmarks/passmark/graph.py index 29a594f..5866799 100644 --- a/quisby/benchmarks/passmark/graph.py +++ b/quisby/benchmarks/passmark/graph.py @@ -112,7 +112,7 @@ def graph_passmark_data(spreadsheetId, range, action): title = "%s : %s" % (range, "Geomean") subtitle = "" - elif "Price/perf" in row: + elif "Price-perf" in row: start_index = index if row_val == 1: row_val = start_index @@ -141,7 +141,7 @@ def graph_passmark_data(spreadsheetId, range, action): "chart": { "spec": { "title": title, - "subtitle": subtitle, + "subtitle": graph_data[1][0], "basicChart": { "chartType": "COMBO", "legendPosition": "RIGHT_LEGEND", diff --git a/quisby/benchmarks/passmark/passmark.py b/quisby/benchmarks/passmark/passmark.py index 14c4bc5..6e67b04 100644 --- a/quisby/benchmarks/passmark/passmark.py +++ b/quisby/benchmarks/passmark/passmark.py @@ -1,8 +1,24 @@ +from itertools import groupby + from scipy.stats import gmean from quisby import custom_logger from quisby.util import read_config from quisby.pricing.cloud_pricing import get_cloud_pricing +import re + +from quisby.util import process_instance, mk_int + + +def extract_prefix_and_number(input_string): + match = re.search(r'^(.*?)(\d+)(.*?)$', input_string) + if match: + prefix = match.group(1) + number = int(match.group(2)) + suffix = match.group(3) + return prefix, number, suffix + return None, None, None + def custom_key(item): cloud_type = read_config("cloud", "cloud_type") @@ -16,6 +32,9 @@ def custom_key(item): instance_type = item[0].split("-")[0] instance_number = int(item[0].split('-')[-1]) return instance_type, instance_number + elif cloud_type == "azure": + instance_type, instance_number, version= extract_prefix_and_number(item[0]) + return instance_type, version, instance_number def calc_price_performance(inst, avg): @@ -33,101 +52,90 @@ def calc_price_performance(inst, avg): return cost_per_hour, price_perf +def group_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature", "machine_type")) + elif cloud_type == "azure": + results = sorted(results, key=lambda x: process_instance(x[1][0], "family", "feature")) + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature")) + elif cloud_type == "gcp": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version","sub_family","feature")) + + +def sort_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + results.sort(key=lambda x: str(process_instance(x[1][0], "family"))) + elif cloud_type == "azure": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "feature"))) + elif cloud_type == "gcp": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "sub_family"))) + + def create_summary_passmark_data(data, OS_RELEASE): - results = [ - ['SYSTEM', "NO_OF_TEST_PROCESSES", "CPU_INTEGER_MATH", "CPU_FLOATINGPOINT_MATH", "CPU_PRIME", "CPU_SORTING", - "CPU_ENCRYPTION", "CPU_COMPRESSION", "CPU_SINGLETHREAD", "CPU_PHYSICS", "CPU_MATRIX_MULT_SSE", "CPU_mm", - "CPU_sse", "CPU_fma", "CPU_avx", "CPU_avx512", "m_CPU_enc_SHA", "m_CPU_enc_AES", "m_CPU_enc_ECDSA", - "ME_ALLOC_S", "ME_READ_S", "ME_READ_L", "ME_WRITE", "ME_LARGE", "ME_LATENCY", "ME_THREADED", "SUMM_CPU", - "SUMM_ME" - ]] - processed_data = None - gmean_data = [] - SYSTEM_GEOMEAN = [] - end_index = 0 - start_index = 0 - system = "" - cost_per_hour, price_per_perf = [], [] - # Add summary data - for index, row in enumerate(data): - if row == [""]: - if processed_data: - inst = processed_data[0] - results.append(processed_data) - gdata = gmean(gmean_data) - SYSTEM_GEOMEAN.append([system, gdata]) + ret_results = [] + results = list(filter(None, data)) + sort_data(results) + results = group_data(results) + for _, items in results: + mac_data = [["System name", "Geomean-" + OS_RELEASE]] + cost_data = [["Cost/Hr"]] + price_perf_data = [["Price-perf", f"Geomean/$-{OS_RELEASE}"]] + items = list(items) + sorted_data = sorted(items, key=lambda x: mk_int(process_instance(x[1][0], "size"))) + cost_per_hour, price_per_perf = [], [] + # Add summary data + for index, row in enumerate(sorted_data): + inst = row[1][0] + gmean_data = [] + for i in range(2, len(row)): try: - cph, pp = calc_price_performance(inst, gdata) + gmean_data.append(float(row[i][1].strip())) except Exception as exc: - custom_logger.error(str(exc)) - continue - price_per_perf.append([inst, pp]) - cost_per_hour.append([inst, cph]) - - processed_data = [] - gmean_data = [] - system = "" - start_index = end_index + 1 - end_index = 0 - elif start_index: - system = row[0] - processed_data.append(system) - end_index = start_index + 1 - start_index = 0 - elif end_index: - if not row[0] == 'NumTestProcesses': - gmean_data.append(float(row[1])) - processed_data.append(row[1]) - - if processed_data: - cph = 0 - pp = 0 - inst = processed_data[0] - results.append(processed_data) - gdata = gmean(gmean_data) - SYSTEM_GEOMEAN.append([system, gdata]) - try: - cph, pp = calc_price_performance(inst, gdata) - except Exception as exc: - custom_logger.error(str(exc)) - price_per_perf.append([inst, pp]) - cost_per_hour.append([inst, cph]) + gmean_data.append(0.0) + gdata = gmean(gmean_data) + try: + cph, pp = calc_price_performance(inst, gdata) + except Exception as exc: + custom_logger.error(str(exc)) + continue - results.append([""]) - results.append(["System name", "Geomean-" + str(OS_RELEASE)]) - sorted_data = sorted(SYSTEM_GEOMEAN, key=custom_key) - for item in sorted_data: - results.append(item) - results.append([""]) - results.append(["Cost/Hr"]) - sorted_data = sorted(cost_per_hour, key=custom_key) - for item in sorted_data: - results.append(item) - results.append([""]) - results.append(["Price/perf", f"Geomean/$-{OS_RELEASE}"]) - sorted_data = sorted(price_per_perf, key=custom_key) - for item in sorted_data: - results.append(item) - return results + mac_data.append([inst, gdata]) + cost_data.append([inst, cph]) + price_perf_data.append([inst, pp]) + ret_results.append([""]) + ret_results.extend(mac_data) + ret_results.append([""]) + ret_results.extend(cost_data) + ret_results.append([""]) + ret_results.extend(price_perf_data) + return ret_results def extract_passmark_data(path, system_name, OS_RELEASE): """""" results = [] + summary_data = [] + server = read_config("server", "name") + result_dir = read_config("server", "result_dir") # Extract data from file try: if path.endswith("results.csv"): with open(path) as file: coremark_results = file.readlines() + sum_path = path.split("/./")[1] + summary_data.append([system_name, "http://" + server + "/results/" + result_dir + "/" + sum_path]) + else: return None except Exception as exc: custom_logger.error(str(exc)) - return None + return None, None for index, data in enumerate(coremark_results): coremark_results[index] = data.strip("\n").split(":") results.append([""]) results.append([system_name]) results.extend(coremark_results) - return results \ No newline at end of file + return [results], summary_data \ No newline at end of file diff --git a/quisby/benchmarks/phoronix/compare.py b/quisby/benchmarks/phoronix/compare.py index 957a65e..c6c6e79 100644 --- a/quisby/benchmarks/phoronix/compare.py +++ b/quisby/benchmarks/phoronix/compare.py @@ -15,8 +15,9 @@ def extract_prefix_and_number(input_string): match = re.search(r'^(.*?)(\d+)(.*?)$', input_string) if match: prefix = match.group(1) - return prefix - return None + suffix = match.group(3) # Extracts the suffix after the number + return prefix, suffix + return None, None def compare_inst(item1, item2): @@ -31,7 +32,7 @@ def compare_inst(item1, item2): return extract_prefix_and_number(item1) == extract_prefix_and_number(item2) -def compare_phoronix_results(spreadsheets, spreadsheetId, test_name, table_name=["System name", "Price/perf"]): +def compare_phoronix_results(spreadsheets, spreadsheetId, test_name, table_name=["System name", "Price-perf"]): values = [] results = [] spreadsheet_name = [] @@ -49,12 +50,13 @@ def compare_phoronix_results(spreadsheets, spreadsheetId, test_name, table_name= for ele in list_2: # Check max throughput if value[0][0] in table_name and ele[0][0] in table_name and value[0][0] == ele[0][0]: - results.append([""]) - for item1 in value: - for item2 in ele: - if item1[0] == item2[0]: - results = merge_lists_alternately(results, item1, item2) - break + if compare_inst(value[1][0], ele[1][0]): + results.append([""]) + for item1 in value: + for item2 in ele: + if item1[0] == item2[0]: + results = merge_lists_alternately(results, item1, item2) + break elif value[0][0] == "Cost/Hr" and ele[0][0] == "Cost/Hr": if compare_inst(value[1][0], ele[1][0]): diff --git a/quisby/benchmarks/phoronix/graph.py b/quisby/benchmarks/phoronix/graph.py index 3352b91..b7a4a88 100644 --- a/quisby/benchmarks/phoronix/graph.py +++ b/quisby/benchmarks/phoronix/graph.py @@ -107,7 +107,7 @@ def graph_phoronix_data(spreadsheetId, range, action): start_index = index title = "%s : %s" % (range, "Geomean") subtitle="" - elif "Price/perf" in row: + elif "Price-perf" in row: start_index = index title = "%s : %s" % (range, "Price-Performance") subtitle = "Geomean/$" @@ -134,7 +134,7 @@ def graph_phoronix_data(spreadsheetId, range, action): "chart": { "spec": { "title": title, - "subtitle": subtitle, + "subtitle": graph_data[1][0], "basicChart": { "chartType": "COMBO", "legendPosition": "RIGHT_LEGEND", diff --git a/quisby/benchmarks/phoronix/phoronix.py b/quisby/benchmarks/phoronix/phoronix.py index 6ff6a60..0a3e8bb 100644 --- a/quisby/benchmarks/phoronix/phoronix.py +++ b/quisby/benchmarks/phoronix/phoronix.py @@ -1,8 +1,23 @@ +from itertools import groupby + from scipy.stats import gmean from quisby import custom_logger from quisby.util import read_config from quisby.pricing.cloud_pricing import get_cloud_pricing +import re + +from quisby.util import process_instance, mk_int + + +def extract_prefix_and_number(input_string): + match = re.search(r'^(.*?)(\d+)(.*?)$', input_string) + if match: + prefix = match.group(1) + number = int(match.group(2)) + suffix = match.group(3) + return prefix, number, suffix + return None, None, None def custom_key(item): @@ -17,6 +32,9 @@ def custom_key(item): instance_type = item[0].split("-")[0] instance_number = int(item[0].split('-')[-1]) return instance_type, instance_number + elif cloud_type == "azure": + instance_type, instance_number, version= extract_prefix_and_number(item[0]) + return instance_type, version, instance_number def calc_price_performance(inst, avg): @@ -34,81 +52,80 @@ def calc_price_performance(inst, avg): return cost_per_hour, price_perf +def group_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature", "machine_type")) + elif cloud_type == "azure": + results = sorted(results, key=lambda x: process_instance(x[1][0], "family", "feature")) + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature")) + elif cloud_type == "gcp": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version","sub_family","feature")) + + +def sort_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + results.sort(key=lambda x: str(process_instance(x[1][0], "family"))) + elif cloud_type == "azure": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "feature"))) + elif cloud_type == "gcp": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "sub_family"))) + def create_summary_phoronix_data(data, OS_RELEASE): - results = [] - processed_data = None - gmean_data = [] - SYSTEM_GEOMEAN = [] - end_index = 0 - start_index = 0 - system = "" - cost_per_hour, price_per_perf = [], [] - # Add summary data - for index, row in enumerate(data): - if row == [""]: - if processed_data: - inst = processed_data[0] - gdata = gmean(gmean_data) - SYSTEM_GEOMEAN.append([system, gdata]) + ret_results = [] + + results = list(filter(None, data)) + sort_data(results) + results = group_data(results) + for _, items in results: + mac_data = [["System name", "Geomean-" + OS_RELEASE]] + cost_data = [["Cost/Hr"]] + price_perf_data = [["Price-perf",f"Geomean/$-{OS_RELEASE}"]] + items = list(items) + sorted_data = sorted(items, key=lambda x: mk_int(process_instance(x[1][0], "size"))) + cost_per_hour, price_per_perf = [], [] + # Add summary data + for index, row in enumerate(sorted_data): + inst = row[1][0] + gmean_data=[] + for i in range(2,len(row)): try: - cph, pp = calc_price_performance(inst, gdata) + gmean_data.append(float(row[i][1].strip())) except Exception as exc: - custom_logger.error(str(exc)) - continue - price_per_perf.append([inst, pp]) - cost_per_hour.append([inst, cph]) - processed_data = [] - gmean_data = [] - system = "" - start_index = end_index + 1 - end_index = 0 - elif start_index: - system = row[0] - processed_data.append(system) - end_index = start_index + 1 - start_index = 0 - elif end_index: - gmean_data.append(float(row[1])) - if processed_data: - cph = 0 - pp = 0 - inst = processed_data[0] - gdata = gmean(gmean_data) - SYSTEM_GEOMEAN.append([system, gdata]) - try: - cph, pp = calc_price_performance(inst, gdata) - except Exception as exc: - custom_logger.error(str(exc)) - price_per_perf.append([inst, pp]) - - cost_per_hour.append([inst, cph]) + gmean_data.append(0.0) + gdata = gmean(gmean_data) + try: + cph, pp = calc_price_performance(inst, gdata) + except Exception as exc: + custom_logger.error(str(exc)) + continue - results.append([""]) - results.append(["System name", "Geomean-" + str(OS_RELEASE)]) - sorted_data = sorted(SYSTEM_GEOMEAN, key=custom_key) - for item in sorted_data: - results.append(item) - results.append([""]) - results.append(["Cost/Hr"]) - sorted_data = sorted(cost_per_hour, key=custom_key) - for item in sorted_data: - results.append(item) - results.append([""]) - results.append(["Price/perf", f"Geomean/$-{OS_RELEASE}"]) - sorted_data = sorted(price_per_perf, key=custom_key) - for item in sorted_data: - results.append(item) - return results + mac_data.append([inst, gdata]) + cost_data.append([inst, cph]) + price_perf_data.append([inst, pp]) + ret_results.append([""]) + ret_results.extend(mac_data) + ret_results.append([""]) + ret_results.extend(cost_data) + ret_results.append([""]) + ret_results.extend(price_perf_data) + return ret_results def extract_phoronix_data(path, system_name, OS_RELEASE): """""" results = [] + summary_data = [] + server = read_config("server", "name") + result_dir = read_config("server", "result_dir") # Extract data from file try: if path.endswith("results.csv"): with open(path) as file: phoronix_results = file.readlines() + sum_path = path.split("/./")[1] + summary_data.append([system_name, "http://" + server + "/results/" + result_dir + "/" + sum_path]) else: return None except Exception as exc: @@ -120,4 +137,4 @@ def extract_phoronix_data(path, system_name, OS_RELEASE): results.append([""]) results.append([system_name]) results.extend(phoronix_results[1:]) - return results \ No newline at end of file + return [results], summary_data \ No newline at end of file diff --git a/quisby/benchmarks/pyperf/compare.py b/quisby/benchmarks/pyperf/compare.py index 6bb4bc7..ff4e3c4 100644 --- a/quisby/benchmarks/pyperf/compare.py +++ b/quisby/benchmarks/pyperf/compare.py @@ -15,8 +15,9 @@ def extract_prefix_and_number(input_string): match = re.search(r'^(.*?)(\d+)(.*?)$', input_string) if match: prefix = match.group(1) - return prefix - return None + suffix = match.group(3) # Extracts the suffix after the number + return prefix, suffix + return None, None def compare_inst(item1, item2): @@ -31,7 +32,7 @@ def compare_inst(item1, item2): return extract_prefix_and_number(item1) == extract_prefix_and_number(item2) -def compare_pyperf_results(spreadsheets, spreadsheetId, test_name, table_name=["System name","Price/perf"]): +def compare_pyperf_results(spreadsheets, spreadsheetId, test_name, table_name=["System name","Price-perf"]): values = [] results = [] spreadsheet_name = [] @@ -49,13 +50,12 @@ def compare_pyperf_results(spreadsheets, spreadsheetId, test_name, table_name=[" for ele in list_2: # Check max throughput if value[0][0] in table_name and ele[0][0] in table_name and value[0][0] == ele[0][0]: - if value[1][0].split(".")[0] == ele[1][0].split(".")[0]: + if compare_inst(value[1][0], ele[1][0]): results.append([""]) for item1 in value: for item2 in ele: if item1[0] == item2[0]: results = merge_lists_alternately(results, item1, item2) - break elif value[0][0] == "Cost/Hr" and ele[0][0] == "Cost/Hr": if compare_inst(value[1][0], ele[1][0]): @@ -64,7 +64,6 @@ def compare_pyperf_results(spreadsheets, spreadsheetId, test_name, table_name=[" for item2 in ele: if item1[0] == item2[0]: results.append(item1) - break elif value[1][0] == ele[1][0]: if value[0][0] == ele[0][0]: @@ -72,7 +71,7 @@ def compare_pyperf_results(spreadsheets, spreadsheetId, test_name, table_name=[" results.append(value[0]) for item1, item2 in zip(value[1:], ele[1:]): results = merge_lists_alternately(results, item1, item2) - break + try: create_sheet(spreadsheetId, test_name) diff --git a/quisby/benchmarks/pyperf/graph.py b/quisby/benchmarks/pyperf/graph.py index 1e0fc7f..62ad112 100644 --- a/quisby/benchmarks/pyperf/graph.py +++ b/quisby/benchmarks/pyperf/graph.py @@ -110,7 +110,7 @@ def graph_pyperf_data(spreadsheetId, range, action): row_val = start_index title = "%s : %s" % (range, "Geomean") subtitle = "" - elif "Price/perf" in row: + elif "Price-perf" in row: start_index = index if row_val == 1: row_val = start_index @@ -138,7 +138,7 @@ def graph_pyperf_data(spreadsheetId, range, action): "chart": { "spec": { "title": title, - "subtitle": subtitle, + "subtitle": graph_data[1][0], "basicChart": { "chartType": "COMBO", "legendPosition": "RIGHT_LEGEND", diff --git a/quisby/benchmarks/pyperf/pyperf.py b/quisby/benchmarks/pyperf/pyperf.py index 1623bcf..843578a 100644 --- a/quisby/benchmarks/pyperf/pyperf.py +++ b/quisby/benchmarks/pyperf/pyperf.py @@ -1,8 +1,25 @@ +from itertools import groupby + from scipy.stats import gmean from quisby import custom_logger from quisby.util import read_config from quisby.pricing.cloud_pricing import get_cloud_pricing +import re + +from quisby.util import process_instance, mk_int + +from quisby.sheet.sheet_util import append_to_sheet + + +def extract_prefix_and_number(input_string): + match = re.search(r'^(.*?)(\d+)(.*?)$', input_string) + if match: + prefix = match.group(1) + number = int(match.group(2)) + suffix = match.group(3) + return prefix, number, suffix + return None, None, None def custom_key(item): @@ -17,6 +34,9 @@ def custom_key(item): instance_type = item[0].split("-")[0] instance_number = int(item[0].split('-')[-1]) return instance_type, instance_number + elif cloud_type == "azure": + instance_type, instance_number, version= extract_prefix_and_number(item[0]) + return instance_type, version, instance_number def calc_price_performance(inst, avg): @@ -34,101 +54,90 @@ def calc_price_performance(inst, avg): return cost_per_hour, price_perf +def group_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature", "machine_type")) + elif cloud_type == "azure": + results = sorted(results, key=lambda x: process_instance(x[1][0], "family", "feature")) + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature")) + elif cloud_type == "gcp": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version","sub_family","feature")) + + + +def sort_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + results.sort(key=lambda x: str(process_instance(x[1][0], "family"))) + elif cloud_type == "azure": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "feature"))) + elif cloud_type == "gcp": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "sub_family"))) + + def create_summary_pyperf_data(data, OS_RELEASE): - results = [] - processed_data = None - gmean_data = [] - SYSTEM_GEOMEAN = [] - end_index = 0 - start_index = 0 - system = "" - cost_per_hour, price_per_perf = [], [] - # Add summary data - for index, row in enumerate(data): - try: - if row == [""]: - if processed_data: - inst = processed_data[0] - results.append(processed_data) - gdata = gmean(gmean_data) - SYSTEM_GEOMEAN.append([system, gdata]) - try: - cph, pp = calc_price_performance(inst, gdata) - except Exception as exc: - custom_logger.error(str(exc)) - continue - price_per_perf.append([inst, pp]) - cost_per_hour.append([inst, cph]) - - processed_data = [] - gmean_data = [] - system = "" - start_index = end_index + 1 - end_index = 0 - elif start_index: - system = row[0] - processed_data.append(system) - end_index = start_index + 1 - start_index = 0 - elif end_index: - gmean_data.append(float(row[1])) - processed_data.append(row[0] + " :" + row[1]) - if float(row[1] == 0.0): - custom_logger.warning("Value for test: " + row[0] + " is 0.0 for machine " + system) - except Exception as exc: - custom_logger.error(str(exc)) - - if processed_data: - cph = 0 - pp = 0 - inst = processed_data[0] - results.append(processed_data) - gdata = gmean(gmean_data) - SYSTEM_GEOMEAN.append([system, gdata]) - try: - cph, pp = calc_price_performance(inst, gdata) - except Exception as exc: - custom_logger.error(str(exc)) - price_per_perf.append([inst, pp]) - cost_per_hour.append([inst, cph]) - - # results.append(processed_data) - # SYSTEM_GEOMEAN.append([system, gmean(gmean_data)]) - results.append([""]) - results.append(["System name", "Geomean-" + str(OS_RELEASE)]) - sorted_data = sorted(SYSTEM_GEOMEAN, key=custom_key) - for item in sorted_data: - results.append(item) - results.append([""]) - results.append(["Cost/Hr"]) - sorted_data = sorted(cost_per_hour, key=custom_key) - for item in sorted_data: - results.append(item) - results.append([""]) - results.append(["Price/perf", f"Geomean/$-{OS_RELEASE}"]) - sorted_data = sorted(price_per_perf, key=custom_key) - for item in sorted_data: - results.append(item) - return results + ret_results = [] + + results = list(filter(None, data)) + sort_data(results) + results = group_data(results) + for _, items in results: + mac_data = [["System name", "Geomean-" + OS_RELEASE]] + cost_data = [["Cost/Hr"]] + price_perf_data = [["Price-perf",f"Geomean/$-{OS_RELEASE}"]] + items = list(items) + sorted_data = sorted(items, key=lambda x: mk_int(process_instance(x[1][0], "size"))) + cost_per_hour, price_per_perf = [], [] + # Add summary data + for index, row in enumerate(sorted_data): + inst = row[1][0] + gmean_data=[] + for i in range(2, len(row)): + try: + gmean_data.append(float(row[i][1].strip())) + except Exception as exc: + gmean_data.append(0.0) + gdata = gmean(gmean_data) + try: + cph, pp = calc_price_performance(inst, gdata) + except Exception as exc: + custom_logger.error(str(exc)) + continue + + mac_data.append([inst, gdata]) + cost_data.append([inst, cph]) + price_perf_data.append([inst, pp]) + ret_results.append([""]) + ret_results.extend(mac_data) + ret_results.append([""]) + ret_results.extend(cost_data) + ret_results.append([""]) + ret_results.extend(price_perf_data) + return ret_results def extract_pyperf_data(path, system_name, OS_RELEASE): """""" results = [] + server = read_config("server", "name") + result_dir = read_config("server", "result_dir") # Extract data from file + summary_data = [] try: if path: with open(path) as file: pyperf_results = file.readlines() + sum_path = path.split("/./")[1] + summary_data.append([system_name, "http://"+server+"/results/"+result_dir+"/"+sum_path]) else: return None except Exception as exc: custom_logger.error(str(exc)) return None - for index, data in enumerate(pyperf_results): pyperf_results[index] = data.strip("\n").split(":") results.append([""]) results.append([system_name]) - results.extend(pyperf_results[0:]) - return results \ No newline at end of file + results.extend(pyperf_results[1:]) + return [results], summary_data \ No newline at end of file diff --git a/quisby/benchmarks/specjbb/comparison.py b/quisby/benchmarks/specjbb/comparison.py index d02cfd2..db847fc 100644 --- a/quisby/benchmarks/specjbb/comparison.py +++ b/quisby/benchmarks/specjbb/comparison.py @@ -16,8 +16,9 @@ def extract_prefix_and_number(input_string): match = re.search(r'^(.*?)(\d+)(.*?)$', input_string) if match: prefix = match.group(1) - return prefix - return None + suffix = match.group(3) # Extracts the suffix after the number + return prefix, suffix + return None, None def compare_inst(item1, item2): diff --git a/quisby/benchmarks/specjbb/graph.py b/quisby/benchmarks/specjbb/graph.py index b17d0dd..3dc0e75 100644 --- a/quisby/benchmarks/specjbb/graph.py +++ b/quisby/benchmarks/specjbb/graph.py @@ -107,8 +107,17 @@ def graph_specjbb_data(spreadsheetId, range, action): for index, row in enumerate(data): try: - if "Peak" in row or "Peak/$eff" in row: + if "Peak" in row: start_index = index + title = "%s : %s" %(range,row[0]) + subtitle = "Throughput" + left_title = row[1].lower() + + if "Peak/$eff" in row: + start_index = index + title = "%s : %s" % (range, "Price-Performance") + subtitle = "%s" %(row[0]) + left_title = row[1].lower() if start_index: if not row: @@ -130,17 +139,36 @@ def graph_specjbb_data(spreadsheetId, range, action): "addChart": { "chart": { "spec": { - "title": "%s : %s" % (range, graph_data[0][0]), + "title": title, + "subtitle": subtitle+" : "+graph_data[1][0], "basicChart": { "chartType": "COMBO", - "legendPosition": "BOTTOM_LEGEND", + "legendPosition": "RIGHT_LEGEND", "axis": [ - {"position": "BOTTOM_AXIS", "title": ""}, { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, + + "position": "BOTTOM_AXIS", + "title": "System"}, + { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, "position": "LEFT_AXIS", - "title": "Throughput(bops)", + "title": left_title, }, { + "format": { + "bold": True, + "italic": True, + "fontSize": 14 + }, "position": "RIGHT_AXIS", "title": "%Diff", }, @@ -172,8 +200,12 @@ def graph_specjbb_data(spreadsheetId, range, action): "sheetId": sheetId, "rowIndex": GRAPH_ROW_INDEX, "columnIndex": column_count + GRAPH_COL_INDEX, - } - } + }, + "offsetXPixels": 100, + "widthPixels": 600, + "heightPixels": 400 + }, + }, } } diff --git a/quisby/benchmarks/specjbb/specjbb.py b/quisby/benchmarks/specjbb/specjbb.py index 95f1bc2..d24bc6e 100644 --- a/quisby/benchmarks/specjbb/specjbb.py +++ b/quisby/benchmarks/specjbb/specjbb.py @@ -31,15 +31,26 @@ def custom_key(item): return instance_type, instance_number elif cloud_type == "azure": instance_type, instance_number, version= extract_prefix_and_number(item[1][0]) - return instance_type, instance_number + return instance_type,version, instance_number +def group_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature", "machine_type")) + elif cloud_type == "azure": + results = sorted(results, key=lambda x: process_instance(x[1][0], "family", "feature")) + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature")) + elif cloud_type == "gcp": + return groupby(results, key=lambda x: process_instance(x[1][0], "family", "version","sub_family","feature")) + def specjbb_sort_data_by_system_family(results): sorted_result = [] results.sort(key=lambda x: str(process_instance( x[1][0], "family", "version", "feature"))) + for _, items in groupby(results, key=lambda x: process_instance(x[1][0], "family", "version", "feature")): sorted_result.append( sorted(list(items), key=lambda x: mk_int( @@ -61,21 +72,35 @@ def calc_peak_throughput_peak_efficiency(data): peak_efficiency = float(peak_throughput) / float(cost_per_hour) except Exception as exc: custom_logger.debug(str(exc)) - custom_logger.error("Error calculating value !") + custom_logger.error("Error calculating value for :" + data[1][0]) return peak_throughput, cost_per_hour, peak_efficiency + +def sort_data(results): + cloud_type = read_config("cloud", "cloud_type") + if cloud_type == "aws": + results.sort(key=lambda x: str(process_instance(x[1][0], "family"))) + elif cloud_type == "azure": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "feature"))) + elif cloud_type == "gcp": + results.sort(key=lambda x: str(process_instance(x[1][0], "family", "version", "sub_family"))) + def create_summary_specjbb_data(specjbb_data, OS_RELEASE): """""" - results = [] - specjbb_data = specjbb_sort_data_by_system_family(specjbb_data) + results = [] + #specjbb_data = specjbb_sort_data_by_system_family(specjbb_data) + specjbb_data = list(filter(None, specjbb_data)) + sort_data(specjbb_data) + specjbb_data = group_data(specjbb_data) - for items in specjbb_data: + for _,items in specjbb_data: + items = list(items) peak_throughput, cost_per_hour, peak_efficiency = [], [], [] - sorted_data = sorted(items, key=custom_key) + sorted_data = sorted(items, key=lambda x: mk_int(process_instance(x[1][0], "size"))) for item in sorted_data: - results += item + results.extend(item) try: pt, cph, pe = calc_peak_throughput_peak_efficiency(item) except Exception as exc: @@ -92,7 +117,7 @@ def create_summary_specjbb_data(specjbb_data, OS_RELEASE): results.append(["Cost/Hr"]) results += cost_per_hour results.append([""]) - results.append(["Peak/$eff", f"Price/perf-{OS_RELEASE}"]) + results.append(["Peak/$eff", f"Price-perf-{OS_RELEASE}"]) results += peak_efficiency return results @@ -101,17 +126,22 @@ def create_summary_specjbb_data(specjbb_data, OS_RELEASE): def extract_specjbb_data(path, system_name, OS_RELEASE): """""" results = [[""], [system_name]] + summary_data = [] + server = read_config("server", "name") + result_dir = read_config("server", "result_dir") # File read try: if path.endswith(".csv"): with open(path) as csv_file: specjbb_results = list(csv.DictReader(csv_file, delimiter=":")) + sum_path = path.split("/./")[1] + summary_data.append([system_name, "http://" + server + "/results/" + result_dir + "/" + sum_path]) else: return None except Exception as exc: custom_logger.error(str(exc)) - return None + return None, None results.append(["Warehouses", f"Thrput-{OS_RELEASE}"]) for data_dict in specjbb_results[1:]: @@ -120,4 +150,4 @@ def extract_specjbb_data(path, system_name, OS_RELEASE): else: results.append([data_dict["Warehouses"], data_dict["Bops"]]) - return results + return results, summary_data