more new results breakdown

experiments/process_and_plot_data.py

@@ -77,6 +77,310 @@ def process_runtime_results(results_dir: Path, figures_dir: Path):
     # colum order
     #
 
+    runtime_all_df_pivot = runtime_df.copy().pivot_table(
+        index=["model", "dataset"],
+        values="runtime",
+        columns="runtime_type",
+    )
+    runtime_all_df_pivot.reindex(
+        index=["cpp_cpu", "pyg_cpu", "pyg_gpu", "fpga_base", "fpga_par"]
+    )
+    print(runtime_all_df_pivot)
+
+    speedup_all_df = pd.DataFrame()
+    speedup_all_df["fpga_par_speedup_over_pyg_gpu"] = (
+        runtime_all_df_pivot["pyg_gpu"] / runtime_all_df_pivot["fpga_par"]
+    )
+    speedup_all_df["fpga_par_speedup_over_pyg_cpu"] = (
+        runtime_all_df_pivot["pyg_cpu"] / runtime_all_df_pivot["fpga_par"]
+    )
+    speedup_all_df["fpga_par_speedup_over_cpp_cpu"] = (
+        runtime_all_df_pivot["cpp_cpu"] / runtime_all_df_pivot["fpga_par"]
+    )
+
+    speedup_all_df = speedup_all_df.reindex(
+        columns=[
+            "fpga_par_speedup_over_cpp_cpu",
+            "fpga_par_speedup_over_pyg_cpu",
+            "fpga_par_speedup_over_pyg_gpu",
+        ]
+    )
+
+    # rename the columns
+    speedup_all_df = speedup_all_df.rename(
+        columns={
+            "fpga_par_speedup_over_cpp_cpu": "CPP-CPU",
+            "fpga_par_speedup_over_pyg_cpu": "PYG-CPU",
+            "fpga_par_speedup_over_pyg_gpu": "PYG-GPU",
+        }
+    )
+
+    # rename the multiindex
+    speedup_all_df.index = speedup_all_df.index.rename(
+        {"model": "Model", "dataset": "Dataset"}
+    )
+
+    speedup_all_df_latex = speedup_all_df.to_latex(
+        float_format="{:0.2f}x".format,
+        multicolumn_format="c",
+        column_format="c | ccc",
+        caption="FPGA-PAR Runtime speedup over PyG CPU, PyG GPU, and C++ CPU runtimes.",
+        label="tab:runtime_speedup",
+        escape=True,
+        sparsify=True,
+    )
+    print()
+    print(speedup_all_df_latex)
+    print()
+    latex_output_file = figures_dir / "speedup_all.tex"
+    latex_output_file.write_text(speedup_all_df_latex)
+
+    speedup_all_df.to_csv(figures_dir / "speedup_all.csv")
+    speedup_all_df.to_excel(figures_dir / "speedup_all.xlsx")
+
+    # exit()
+
+    speedup_min_max_df = speedup_all_df.copy().groupby(["Model"]).agg(["min", "max"])
+    # merge the min and max columns for each type so that the rows hold a [min, max] list, the multiindex is (implementaion, agg)
+
+    speedup_min_max_df.to_csv(figures_dir / "speedup_min_max.csv")
+    speedup_min_max_df.to_excel(figures_dir / "speedup_min_max.xlsx")
+
+    speedup_min_max_df["fpga_par_speedup_over_cpp_cpu_min_max"] = speedup_min_max_df[
+        [
+            ("CPP-CPU", "min"),
+            ("CPP-CPU", "max"),
+        ]
+    ].values.tolist()
+    speedup_min_max_df["fpga_par_speedup_over_pyg_cpu_min_max"] = speedup_min_max_df[
+        [
+            ("PYG-CPU", "min"),
+            ("PYG-CPU", "max"),
+        ]
+    ].values.tolist()
+    speedup_min_max_df["fpga_par_speedup_over_pyg_gpu_min_max"] = speedup_min_max_df[
+        [
+            ("PYG-GPU", "min"),
+            ("PYG-GPU", "max"),
+        ]
+    ].values.tolist()
+
+    speedup_min_max_df = speedup_min_max_df.drop(
+        columns=[
+            ("CPP-CPU", "min"),
+            ("CPP-CPU", "max"),
+            ("PYG-CPU", "min"),
+            ("PYG-CPU", "max"),
+            ("PYG-GPU", "min"),
+            ("PYG-GPU", "max"),
+        ]
+    )
+
+    # rename the columns
+    speedup_min_max_df = speedup_min_max_df.rename(
+        columns={
+            "fpga_par_speedup_over_cpp_cpu_min_max": "CPP-CPU",
+            "fpga_par_speedup_over_pyg_cpu_min_max": "PYG-CPU",
+            "fpga_par_speedup_over_pyg_gpu_min_max": "PYG-GPU",
+        }
+    )
+
+    speedup_min_max_df.columns = [
+        c[0] for c in speedup_min_max_df.columns.to_flat_index()
+    ]
+
+    def min_max_formatter(x):
+        return f"{x[0]:.2f}-{x[1]:.2f} x"
+
+    speedup_min_max_df_latex = speedup_min_max_df.to_latex(
+        formatters=[min_max_formatter] * 3,
+        multicolumn_format="c",
+        column_format="c | ccc",
+        caption="FPGA-PAR Runtime speedup over PyG CPU, PyG GPU, and C++ CPU runtimes.",
+        label="tab:runtime_speedup",
+        escape=True,
+        sparsify=True,
+    )
+    print()
+    print(speedup_min_max_df_latex)
+    print()
+
+    latex_output_file = figures_dir / "speedup_min_max.tex"
+    latex_output_file.write_text(speedup_min_max_df_latex)
+
+    fig = plt.figure(figsize=(5, 4))
+    ax = fig.add_subplot(1, 1, 1)
+
+    model_rename = {
+        "gcn": "GCN",
+        "gin": "GIN",
+        "pna": "PNA",
+        "sage": "SAGE",
+    }
+
+    models = speedup_min_max_df.index.to_list()
+    models = [model_rename[m] for m in models]
+    implementations = speedup_min_max_df.columns
+
+    colors = ["#95d5b2", "#90e0ef", "#00b4d8"]
+
+    x = np.arange(len(models))  # the label locations
+    width = 0.2  # the width of the bars
+    with_factor = 0.9
+    multiplier = 0
+
+    for i, col in enumerate(implementations):
+        col_values = speedup_min_max_df[col].to_list()
+        min_values = [x[0] for x in col_values]
+        max_values = [x[1] for x in col_values]
+
+        offset = width * multiplier
+        # make a vertical bar plot but start at min_values and end at max_values
+        ax.bar(
+            x + offset,
+            max_values,
+            width * with_factor,
+            bottom=min_values,
+            label=col,
+            color=colors[i],
+        )
+        multiplier += 1
+
+    ax.set_xlabel("GNN Model")
+    ax.set_xticks(x + width, models)
+
+    # get defualt xlim
+    xlim_min, xlim_max = ax.get_xlim()
+    ax.set_xlim(xlim_min, xlim_max)
+    ax.hlines(1, xlim_min, xlim_max, colors="gray", linestyles="dashed")
+
+    ax.set_ylabel("Speedup ($\\times$)")
+    ax.set_ylim(0.25, 21)
+    ax.set_yticks(np.arange(1, 21, 1))
+
+    # draw horizontal grid lines
+    ax.set_axisbelow(True)
+    ax.yaxis.grid(True, linestyle="-", which="major", color="lightgrey", alpha=0.5)
+
+    ax.legend()
+    ax.set_title("FPGA-PAR Runtime Speedup")
+
+    plt.tight_layout()
+
+    figure_path = figures_dir / "speedup_min_max.png"
+    plt.savefig(figure_path, dpi=300)
+    plt.close()
+
+    # Add some text for labels, title and custom x-axis tick labels, etc.
+    exit()
+
+    # print(runtime_df_pivot_range)
+    # print(runtime_df_pivot_range.columns)
+
+    # speedup_df = pd.DataFrame()
+    # # speedup_df["model"] = runtime_df_pivot_range.index.get_level_values(0)
+    # # speedup_df["dataset"] = runtime_df_pivot_range.index.get_level_values(1)
+    # speedup_df["fpga_par_speedup_over_pyg_cpu_min"] = (
+    #     runtime_df_pivot_range["min"]["pyg_cpu"]
+    #     / runtime_df_pivot_range["max"]["fpga_par"]
+    # )
+    # speedup_df["fpga_par_speedup_over_pyg_gpu_min"] = (
+    #     runtime_df_pivot_range["min"]["pyg_gpu"]
+    #     / runtime_df_pivot_range["max"]["fpga_par"]
+    # )
+    # speedup_df["fpga_par_speedup_over_cpp_cpu_min"] = (
+    #     runtime_df_pivot_range["min"]["cpp_cpu"]
+    #     / runtime_df_pivot_range["max"]["fpga_par"]
+    # )
+    # speedup_df["fpga_par_speedup_over_pyg_cpu_max"] = (
+    #     runtime_df_pivot_range["max"]["pyg_cpu"]
+    #     / runtime_df_pivot_range["min"]["fpga_par"]
+    # )
+    # speedup_df["fpga_par_speedup_over_pyg_gpu_max"] = (
+    #     runtime_df_pivot_range["max"]["pyg_gpu"]
+    #     / runtime_df_pivot_range["min"]["fpga_par"]
+    # )
+    # speedup_df["fpga_par_speedup_over_cpp_cpu_max"] = (
+    #     runtime_df_pivot_range["max"]["cpp_cpu"]
+    #     / runtime_df_pivot_range["min"]["fpga_par"]
+    # )
+
+    # speedup_df = speedup_df.reindex(
+    #     columns=[
+    #         "fpga_par_speedup_over_cpp_cpu_min",
+    #         "fpga_par_speedup_over_cpp_cpu_max",
+    #         "fpga_par_speedup_over_pyg_cpu_min",
+    #         "fpga_par_speedup_over_pyg_cpu_max",
+    #         "fpga_par_speedup_over_pyg_gpu_min",
+    #         "fpga_par_speedup_over_pyg_gpu_max",
+    #     ]
+    # )
+
+    # # print(speedup_df)
+
+    # # speedup_df = speedup_df.reorder_levels(["dataset", "model"], axis=0)
+    # print(speedup_df)
+    # # combine the min and max columns for each type so that the rows hold a [min, max] list
+    # speedup_df["fpga_par_speedup_over_cpp_cpu_min_max"] = speedup_df[
+    #     [
+    #         "fpga_par_speedup_over_cpp_cpu_min",
+    #         "fpga_par_speedup_over_cpp_cpu_max",
+    #     ]
+    # ].values.tolist()
+    # speedup_df["fpga_par_speedup_over_pyg_cpu_min_max"] = speedup_df[
+    #     [
+    #         "fpga_par_speedup_over_pyg_cpu_min",
+    #         "fpga_par_speedup_over_pyg_cpu_max",
+    #     ]
+    # ].values.tolist()
+    # speedup_df["fpga_par_speedup_over_pyg_gpu_min_max"] = speedup_df[
+    #     [
+    #         "fpga_par_speedup_over_pyg_gpu_min",
+    #         "fpga_par_speedup_over_pyg_gpu_max",
+    #     ]
+    # ].values.tolist()
+
+    # speedup_df = speedup_df.drop(
+    #     columns=[
+    #         "fpga_par_speedup_over_cpp_cpu_min",
+    #         "fpga_par_speedup_over_cpp_cpu_max",
+    #         "fpga_par_speedup_over_pyg_cpu_min",
+    #         "fpga_par_speedup_over_pyg_cpu_max",
+    #         "fpga_par_speedup_over_pyg_gpu_min",
+    #         "fpga_par_speedup_over_pyg_gpu_max",
+    #     ]
+    # )
+
+    # # rename the columns
+    # speedup_df = speedup_df.rename(
+    #     columns={
+    #         "fpga_par_speedup_over_cpp_cpu_min_max": "CPP-CPU",
+    #         "fpga_par_speedup_over_pyg_cpu_min_max": "PYG-CPU",
+    #         "fpga_par_speedup_over_pyg_gpu_min_max": "PYG-GPU",
+    #     }
+    # )
+
+    # # rename the multiindex
+    # speedup_df.index = speedup_df.index.rename({"model": "Model", "dataset": "Dataset"})
+
+    # print(speedup_df)
+
+    # min_max_formatter = lambda x: f"{x[0]:.2f}-{x[1]:.2f} x"
+
+    # latex_table = speedup_df.to_latex(
+    #     formatters=[min_max_formatter] * 3,
+    #     multicolumn_format="c",
+    #     column_format="c | ccc",
+    #     caption="FPGA-PAR Runtime speedup over PyG CPU, PyG GPU, and C++ CPU runtimes.",
+    #     label="tab:runtime_speedup",
+    #     escape=True,
+    #     sparsify=True,
+    # )
+    # print()
+    # print(latex_table)
+    # print()
+    # exit()
+
     # pivot table
     # rows are models
     # cols are runtime_type
@@ -692,6 +996,7 @@ def process_energy_results(results_dir, results_batch_dir, figures_dir):
     os.makedirs(FIGURES_TESTING_DIR, exist_ok=True)
 
     process_runtime_results(RESULTS_TESTING_DIR, FIGURES_TESTING_DIR)
-    process_resource_results(RESULTS_TESTING_DIR, FIGURES_TESTING_DIR)
+    # process_resource_results(RESULTS_TESTING_DIR, FIGURES_TESTING_DIR)
+
     # process_batch_results(RESULTS_TESTING_DIR, RESULTS_BATCH_DIR, FIGURES_TESTING_DIR)
     # process_energy_results(RESULTS_TESTING_DIR, RESULTS_BATCH_DIR, FIGURES_TESTING_DIR)

figures_testing/speedup_all.csv

@@ -0,0 +1,21 @@
+Model,Dataset,CPP-CPU,PYG-CPU,PYG-GPU
+gcn,esol,1.9937358837309347,5.147110029196618,6.334349798182242
+gcn,freesolv,1.9098776168422478,7.401511034010365,9.259624498046168
+gcn,hiv,2.0203267846798267,3.397796437103276,4.135333139342365
+gcn,lipo,1.747228435347701,2.3590388328060836,2.820245276257747
+gcn,qm9,2.1736785368038882,5.92527903693377,7.287491654971861
+gin,esol,2.1467366072537857,3.552068233614654,3.8598840706719524
+gin,freesolv,2.085991471370393,5.139160344071845,5.718004808538278
+gin,hiv,2.158102811667815,2.3571949451984393,2.5261002902425536
+gin,lipo,1.9056616280285954,1.6471992615416355,1.7747447731846588
+gin,qm9,2.1380333164793552,3.8379066913756588,4.233899833858178
+pna,esol,10.796423497838779,3.9267798659114996,4.045672911153037
+pna,freesolv,10.308066308516604,5.417842257735578,5.872684070279726
+pna,hiv,10.895955272834042,2.71721611334042,2.7028066780996456
+pna,lipo,9.572408857330311,1.9736050183303624,1.9345196675340388
+pna,qm9,11.171620516999319,4.336826249637138,4.6518522821780195
+sage,esol,4.674175131959364,4.166270404389831,4.77948508950252
+sage,freesolv,4.514482734198377,5.941819548623934,6.833948695974059
+sage,hiv,4.5240052089427305,2.7566031577758854,3.178234000595889
+sage,lipo,4.175718946716845,1.937202204957698,2.3755069336794996
+sage,qm9,4.717928478728842,4.630751782779955,5.471061670105085

figures_testing/speedup_all.tex

@@ -0,0 +1,35 @@
+\begin{table}
+\caption{FPGA-PAR Runtime speedup over PyG CPU, PyG GPU, and C++ CPU runtimes.}
+\label{tab:runtime_speedup}
+\begin{tabular}{c | ccc}
+\toprule
+ &  & CPP-CPU & PYG-CPU & PYG-GPU \\
+Model & Dataset &  &  &  \\
+\midrule
+\multirow[t]{5}{*}{gcn} & esol & 1.99x & 5.15x & 6.33x \\
+ & freesolv & 1.91x & 7.40x & 9.26x \\
+ & hiv & 2.02x & 3.40x & 4.14x \\
+ & lipo & 1.75x & 2.36x & 2.82x \\
+ & qm9 & 2.17x & 5.93x & 7.29x \\
+\cline{1-5}
+\multirow[t]{5}{*}{gin} & esol & 2.15x & 3.55x & 3.86x \\
+ & freesolv & 2.09x & 5.14x & 5.72x \\
+ & hiv & 2.16x & 2.36x & 2.53x \\
+ & lipo & 1.91x & 1.65x & 1.77x \\
+ & qm9 & 2.14x & 3.84x & 4.23x \\
+\cline{1-5}
+\multirow[t]{5}{*}{pna} & esol & 10.80x & 3.93x & 4.05x \\
+ & freesolv & 10.31x & 5.42x & 5.87x \\
+ & hiv & 10.90x & 2.72x & 2.70x \\
+ & lipo & 9.57x & 1.97x & 1.93x \\
+ & qm9 & 11.17x & 4.34x & 4.65x \\
+\cline{1-5}
+\multirow[t]{5}{*}{sage} & esol & 4.67x & 4.17x & 4.78x \\
+ & freesolv & 4.51x & 5.94x & 6.83x \\
+ & hiv & 4.52x & 2.76x & 3.18x \\
+ & lipo & 4.18x & 1.94x & 2.38x \\
+ & qm9 & 4.72x & 4.63x & 5.47x \\
+\cline{1-5}
+\bottomrule
+\end{tabular}
+\end{table}

figures_testing/speedup_min_max.csv

@@ -0,0 +1,7 @@
+,CPP-CPU,CPP-CPU,PYG-CPU,PYG-CPU,PYG-GPU,PYG-GPU
+,min,max,min,max,min,max
+Model,,,,,,
+gcn,1.747228435347701,2.1736785368038882,2.3590388328060836,7.401511034010365,2.820245276257747,9.259624498046168
+gin,1.9056616280285954,2.158102811667815,1.6471992615416355,5.139160344071845,1.7747447731846588,5.718004808538278
+pna,9.572408857330311,11.171620516999319,1.9736050183303624,5.417842257735578,1.9345196675340388,5.872684070279726
+sage,4.175718946716845,4.717928478728842,1.937202204957698,5.941819548623934,2.3755069336794996,6.833948695974059