plot_accuracies.py

import pandas as pd
import matplotlib.pyplot as plt
import os


def get_csv_case_name(csv_path: str):
    stripped = os.path.basename(csv_path)
    # remove _results.csv
    stripped = stripped[:-12]
    return stripped


def get_dfs(csvs, csv_substr: str, exclude_vals: dict):
    dfs = []
    chosen_csvs = []
    for csv in csvs:
        if csv_substr in csv:
            dfs.append(exclude_cols(csv, exclude_vals))
            chosen_csvs.append(csv)
    return dfs, chosen_csvs


def exclude_cols(csv: str, exclude_vals: dict):
    df = pd.read_csv(csv)
    for col_name in exclude_vals.keys():
        if col_name in df.columns:
            for val in exclude_vals[col_name]:
                df = df.loc[df[col_name] != val]
    return df


def get_ovr_accuracy(df: pd.DataFrame):
    runs = df.shape[0]
    true_consensus = df.loc[df['start'] == df['consensus']]
    num_true = len(true_consensus)
    return num_true / runs


def accuracy_by_weight(df: pd.DataFrame):
    weights1 = df.loc[df['weights'] == '0.43 0.57'].reset_index()
    weights2 = df.loc[df['weights'] == '0.46 0.54'].reset_index()
    weights3 = df.loc[df['weights'] == '0.49 0.51'].reset_index()

    weights1_acc = \
        weights1.loc[weights1['start'] == weights1['consensus']].shape[0] / \
        weights1.shape[0]
    weights2_acc = \
        weights2.loc[weights2['start'] == weights2['consensus']].shape[0] / \
        weights2.shape[0]
    weights3_acc = \
        weights3.loc[weights3['start'] == weights3['consensus']].shape[0] / \
        weights3.shape[0]

    weights1_mean_iters = weights1['iterations'].mean()
    weights2_mean_iters = weights2['iterations'].mean()
    weights3_mean_iters = weights3['iterations'].mean()

    return [weights1_acc, weights2_acc, weights3_acc, weights1_mean_iters,
            weights2_mean_iters, weights3_mean_iters]


def accuracies_by_column_vals(df: pd.DataFrame, column: str, vals: list):
    # all overwhelm dfs have same unique column vals
    cases = []
    for val in vals:
        cases.append(df.loc[df[column] == val].reset_index())
    accuracies = []
    for sub_df in cases:
        if sub_df.shape[0] != 0:
            acc = sub_df.loc[sub_df['start'] == sub_df['consensus']].shape[0] / \
                  sub_df.shape[0]
        else:
            acc = 0
        accuracies.append(acc)
    return accuracies


def plot_dark_cases(csvs: list, exclude_vals: dict, colors: list, fig_dir: str):
    dfs, chosen_csvs = get_dfs(csvs, "dark", exclude_vals)
    # all overwhelm dfs have same unique column vals
    accuracies = {}
    for df, csv in zip(dfs, chosen_csvs):
        key = get_csv_case_name(csv)
        accuracies[key] = accuracies_by_column_vals(df,
                                                    "max_dark_steps",
                                                    [8, 12, 16])
    fig, ax = plt.subplots(figsize=(6, 6))
    width = 0.2
    x = 0
    for acc_set in accuracies.keys():
        ax.bar(x, accuracies[acc_set][0], width=width,
               color=colors[0], edgecolor='black')
        ax.bar(x + width + 0.05, accuracies[acc_set][1], width=width,
               color=colors[1], edgecolor='black')
        ax.bar(x + (2 * width) + 0.1, accuracies[acc_set][2], width=width,
               color=colors[2], edgecolor='black')
        x += 1

    xticks = ['Dark Owhelm', 'Dark']

    ax.set_xticks([i + 0.25 for i in range(len(xticks))])
    ax.set_xticklabels(xticks, fontsize=9)
    plt.title('Accuracy of Consensus by Max Dark Steps')
    plt.ylabel('Accuracy')
    ax.legend(labels=[8, 12, 16], loc='upper left', fontsize=7,
              title="Max Steps")
    plt.ylim(0.5, 0.85)
    plt.tight_layout()
    plt.savefig(os.path.join(fig_dir, "dark_case.png"), dpi=300)


def plot_teleport_cases(csvs: list, exclude_vals: dict, colors: list,
                        fig_dir: str):
    dfs, chosen_csvs = get_dfs(csvs, "teleport", exclude_vals)
    accuracies = {}
    for df, csv in zip(dfs, chosen_csvs):
        key = get_csv_case_name(csv)
        accuracies[key] = accuracies_by_column_vals(df, "max_teleport_distance",
                                                    [20, 40, 60])
    fig, ax = plt.subplots(figsize=(6, 6))
    width = 0.2
    x = 0
    for acc_set in accuracies.keys():
        ax.bar(x, accuracies[acc_set][0], width=width,
               color=colors[0], edgecolor='black')
        ax.bar(x + width + 0.05, accuracies[acc_set][1], width=width,
               color=colors[1], edgecolor='black')
        ax.bar(x + (2 * width) + 0.1, accuracies[acc_set][2], width=width,
               color=colors[2], edgecolor='black')
        x += 1

    xticks = ['T.F. Owhelm', 'T.F.', 'T. Owhelm', 'T.']

    ax.set_xticks([i + 0.25 for i in range(len(xticks))])
    ax.set_xticklabels(xticks, fontsize=9)
    plt.title('Accuracy of Consensus by Max Teleport Distance')
    plt.ylabel('Accuracy')
    ax.legend(labels=["20", "40", "60"], loc='upper left', fontsize=7,
              title="Max Distance")
    plt.ylim(0.5, 0.85)
    plt.tight_layout()
    plt.savefig(os.path.join(fig_dir, "accuracies_by_teleport_dist.png"),
                dpi=300)


def plot_overwhelm_cases(csvs: list, exclude_vals: dict, colors: list,
                         fig_dir: str):
    dfs, chosen_csvs = get_dfs(csvs, "overwhelm", exclude_vals)
    # all overwhelm dfs have same unique column vals
    scalar_accuracies = {}
    oa_accuracies = {}
    for df, csv in zip(dfs, chosen_csvs):
        key = get_csv_case_name(csv)
        scalar_accuracies[key] = accuracies_by_column_vals(df,
                                                           "overwhelm_radius_scaler",
                                                           [0.5, 1.5])
        oa_accuracies[key] = accuracies_by_column_vals(df, "overwhelm", [4, 8])
    # by radius scalar
    fig, ax = plt.subplots(figsize=(6, 6))
    width = 0.2
    x = 0
    for acc_set in scalar_accuracies.keys():
        ax.bar(x, scalar_accuracies[acc_set][0], width=width,
               color=colors[0], edgecolor='black')
        ax.bar(x + width + 0.05, scalar_accuracies[acc_set][1], width=width,
               color=colors[1], edgecolor='black')
        x += 1

    xticks = ['Dark Owhelm', 'T.F. Owhelm', 'T. Owhelm']

    ax.set_xticks([i + 0.15 for i in range(len(xticks))])
    ax.set_xticklabels(xticks, fontsize=9)
    plt.title('Accuracy of Consensus by Radius Scalar')
    plt.ylabel('Accuracy')
    ax.legend(labels=["0.5", "1.5"], loc='upper left', fontsize=7,
              title="Radius Scalar")
    plt.ylim(0.5, 0.85)
    plt.tight_layout()
    plt.savefig(os.path.join(fig_dir, "accuracies_overwhelm_radius_scaler.png"),
                dpi=300)

    fig, ax = plt.subplots(figsize=(6, 6))
    width = 0.2
    x = 0
    for acc_set in oa_accuracies.keys():
        ax.bar(x, oa_accuracies[acc_set][0], width=width,
               color=colors[0], edgecolor='black')
        ax.bar(x + width + 0.05, oa_accuracies[acc_set][1], width=width,
               color=colors[1], edgecolor='black')
        x += 1

    xticks = ['Dark Owhelm', 'T.F. Owhelm', 'T. Owhelm']
    ax.set_xticks([i + 0.15 for i in range(len(xticks))])
    ax.set_xticklabels(xticks, fontsize=9)
    plt.title('Accuracy of Consensus by Overwhelm Agents')
    plt.ylabel('Accuracy')
    ax.legend(labels=["4", "8"], loc='upper left', fontsize=7,
              title="Overwhelm Agents")
    plt.ylim(0.5, 0.85)
    plt.tight_layout()
    plt.savefig(os.path.join(fig_dir, "accuracies_overwhelm_oa.png"), dpi=300)


def plot_radii_cases(csvs: list, exclude_vals: dict, colors: list,
                     fig_dir: str):
    # include all csvs
    dfs, chosen_csvs = get_dfs(csvs, ".csv", exclude_vals)
    # all overwhelm dfs have same unique column vals
    radii_accuracies = {}
    for df, csv in zip(dfs, chosen_csvs):
        key = get_csv_case_name(csv)
        radii_accuracies[key] = accuracies_by_column_vals(df, "radius",
                                                          [20, 40, 60])
    fig, ax = plt.subplots(figsize=(6, 6))
    width = 0.2
    x = 0
    for acc_set in radii_accuracies.keys():
        ax.bar(x, radii_accuracies[acc_set][0], width=width,
               color=colors[0], edgecolor='black')
        ax.bar(x + width + 0.05, radii_accuracies[acc_set][1], width=width,
               color=colors[1], edgecolor='black')
        ax.bar(x + (2 * width) + 0.1, radii_accuracies[acc_set][2], width=width,
               color=colors[2], edgecolor='black')
        x += 1

    xticks = ['Dark Owhelm', 'Dark', 'T.F. Owhelm',
              'T.F.', 'T. Owhelm', 'T.', 'Vanilla']

    ax.set_xticks([i + 0.25 for i in range(len(xticks))])
    ax.set_xticklabels(xticks, fontsize=9, rotation=90)
    plt.title('Accuracy of Consensus by Radii')
    plt.ylabel('Accuracy')
    ax.legend(labels=["20", "40", "60"], loc='upper left', fontsize=7,
              title="Radius")
    plt.ylim(0.5, 0.85)
    plt.tight_layout()
    plt.savefig(os.path.join(fig_dir, 'accuracies_by_radii.png'), dpi=300)


def get_avg_iterations_consensus(df: pd.DataFrame):
    consensus_df = df.loc[df['consensus'] != "None"]
    return consensus_df['iterations'].mean()


def get_avg_iterations_by_weight(df: pd.DataFrame):
    consensus_df = df.loc[df['consensus'] != "None"]
    weights1 = consensus_df.loc[df['weights'] == '0.43 0.57']
    weights2 = consensus_df.loc[df['weights'] == '0.46 0.54']
    weights3 = consensus_df.loc[df['weights'] == '0.49 0.51']
    ls = [weights1, weights2, weights3]
    return [get_avg_iterations_consensus(x) for x in ls]


def plot_overall_accuracy(csvs: list, exclude_vals: dict, fig_dir: str):
    dfs, chosen_csvs = get_dfs(csvs, ".csv", exclude_vals)
    accuracies = {}
    for df, csv in zip(dfs, chosen_csvs):
        key = get_csv_case_name(csv)
        accuracies[key] = get_ovr_accuracy(df)

    # Plot accuracies
    xticks = ['Dark Owhelm', 'Dark', 'T.F. Owhelm',
              'T.F.', 'T. Owhelm', 'T.', 'Vanilla']
    fig, ax = plt.subplots(figsize=(6, 6))
    ax.bar(accuracies.keys(), accuracies.values())
    ax.set_xticks([i + 0.05 for i in range(len(xticks))])
    ax.set_xticklabels(xticks, fontsize=9, rotation=90)
    ax.margins(0.01)
    plt.title('Accuracy of Consensus by Configuration')
    plt.ylabel('Accuracy')
    plt.ylim(0.5, 0.85)
    plt.tight_layout()
    plt.savefig(os.path.join(fig_dir, "ovr_accuracies.png"), dpi=300)


def plot_accuracy_by_weight(csvs: list, exclude_vals: dict, colors: list,
                            fig_dir: str):
    dfs, chosen_csvs = get_dfs(csvs, ".csv", exclude_vals)
    accuracies_by_weight = {}
    for df, csv in zip(dfs, chosen_csvs):
        key = get_csv_case_name(csv)
        accuracies_by_weight[key] = accuracy_by_weight(df)
    # Plot accuracies by weight
    fig, ax = plt.subplots(figsize=(6, 6))
    width = 0.2
    x = 0
    for acc_set in accuracies_by_weight.keys():
        ax.bar(x, accuracies_by_weight[acc_set][0], width=width,
               color=colors[0], edgecolor='black')
        ax.bar(x + width + 0.05, accuracies_by_weight[acc_set][1], width=width,
               color=colors[1], edgecolor='black')
        ax.bar(x + (2 * width) + 0.1, accuracies_by_weight[acc_set][2],
               width=width,
               color=colors[2], edgecolor='black')
        x += 1

    xticks = ['Dark Owhelm', 'Dark', 'T.F. Owhelm',
              'T.F.', 'T. Owhelm', 'T.', 'Vanilla']

    ax.set_xticks([i + 0.25 for i in range(len(xticks))])
    ax.set_xticklabels(xticks, fontsize=9, rotation=90)
    ax.margins(0.01)
    plt.title('Accuracy of Consensus by Weight')
    plt.ylabel('Accuracy')
    ax.legend(labels=["0.43", "0.46", "0.49"], loc='upper left', fontsize=7,
              title="Weight")
    plt.ylim(0.5, 0.85)
    plt.tight_layout()
    plt.savefig(os.path.join(fig_dir, "accuracies_by_weight.png"), dpi=300)


def plot_avg_iterations_consensus(csvs: list, exclude_vals: dict,
                                  colors: list, fig_dir: str):
    dfs, chosen_csvs = get_dfs(csvs, ".csv", exclude_vals)
    avg_iterations = {}
    for df, csv in zip(dfs, chosen_csvs):
        key = get_csv_case_name(csv)
        avg_iterations[key] = get_avg_iterations_by_weight(df)

    # Plot accuracies
    fig, ax = plt.subplots(figsize=(6, 6))
    width = 0.2
    x = 0
    for acc_set in avg_iterations.keys():
        ax.bar(x, avg_iterations[acc_set][0], width=width,
               color=colors[0], edgecolor='black')
        ax.bar(x + width + 0.05, avg_iterations[acc_set][1], width=width,
               color=colors[1], edgecolor='black')
        ax.bar(x + (2 * width) + 0.1, avg_iterations[acc_set][2],
               width=width,
               color=colors[2], edgecolor='black')
        x += 1

    xticks = ['Dark Owhelm', 'Dark', 'T.F. Owhelm',
              'T.F.', 'T. Owhelm', 'T.', 'Vanilla']

    ax.set_xticks([i + 0.25 for i in range(len(xticks))])
    ax.set_xticklabels(xticks, fontsize=9, rotation=90)
    ax.margins(0.01)
    plt.title('Average Iterations to Consensus')
    plt.ylabel('Average Iterations to Converge')
    ax.legend(labels=["0.43", "0.46", "0.49"], loc='upper left', fontsize=7,
              title="Weight")
    plt.ylim(250, 500)
    plt.tight_layout()
    plt.savefig(os.path.join(fig_dir, "avg_iterations.png"), dpi=300)


def main():
    results_dir = 'results/'
    fig_dir = 'figures/standard'
    os.makedirs(fig_dir, exist_ok=True)
    colors = ['#5be7a9', '#f8fe85', '#ffbd67']
    csvs = os.listdir(results_dir)
    exclude_vals = {
        "radius": [5, 10]}  # small radii never converge for any case
    # filter out non-csv files
    csvs = [x for x in csvs if x.endswith(".csv")]
    for i in range(len(csvs)):
        csvs[i] = os.path.join(results_dir, csvs[i])
    plot_overwhelm_cases(csvs, exclude_vals, colors, fig_dir)
    plot_radii_cases(csvs, exclude_vals, colors, fig_dir)
    plot_radii_cases(csvs, exclude_vals, colors, fig_dir)
    plot_overall_accuracy(csvs, exclude_vals, fig_dir)
    plot_accuracy_by_weight(csvs, exclude_vals, colors, fig_dir)
    plot_avg_iterations_consensus(csvs, exclude_vals, colors, fig_dir)
    plot_dark_cases(csvs, exclude_vals, colors, fig_dir)
    plot_teleport_cases(csvs, exclude_vals, colors, fig_dir)


if __name__ == "__main__":
    main()