Skip to content

Commit

Permalink
refactor imputation benchmark results
Browse files Browse the repository at this point in the history
  • Loading branch information
OnnoKampman committed Jun 1, 2024
1 parent 7c5b803 commit af7ad30
Show file tree
Hide file tree
Showing 315 changed files with 17 additions and 252 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@
# Get likelihood of observed data at test locations under predicted covariance matrices.
test_log_likelihood = get_test_log_likelihood(
predicted_covariance_structure=test_locations_predicted_covariance_structure,
y_test=y_test
y_test=y_test,
)
test_likelihoods_df.loc[i_trial, covs_type] = test_log_likelihood

Expand All @@ -86,7 +86,7 @@

likelihoods_filename = f'{data_split:s}_{noise_type:s}_likelihoods_{model_name:s}.csv'
test_likelihoods_savedir = os.path.join(
cfg['git-results-basedir'], 'imputation_benchmark'
cfg['git-results-basedir'], noise_type, data_split, 'imputation_benchmark'
)
if not os.path.exists(test_likelihoods_savedir):
os.makedirs(test_likelihoods_savedir)
Expand Down
18 changes: 11 additions & 7 deletions benchmarks/fmri/sim/imputation_benchmark/plot_LEOO_likelihoods.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,9 @@ def plot_likelihoods_raincloud(


def _plot_all_covs_structures_bar(
config_dict: dict, test_likelihoods_mean_df: pd.DataFrame, test_likelihoods_sem_df: pd.DataFrame
config_dict: dict,
test_likelihoods_mean_df: pd.DataFrame,
test_likelihoods_sem_df: pd.DataFrame,
) -> None:
"""
Plot joint test likelihoods for all methods for all covariance structures.
Expand All @@ -82,18 +84,21 @@ def _plot_all_covs_structures_bar(
data_split = 'LEOO' # leave-every-other-out

data_set_name = sys.argv[1] # 'd2', 'd3d', or 'd3s'
experiment_data = sys.argv[2] # e.g. 'N0200_T0003'
experiment_data = sys.argv[2] # 'Nxxxx_Txxxx'

cfg = get_config_dict(
data_set_name=data_set_name,
experiment_data=experiment_data,
hostname=socket.gethostname()
)
num_trials = int(experiment_data[-4:])
test_likelihoods_savedir = os.path.join(cfg['git-results-basedir'], 'imputation_benchmark')

for noise_type in cfg['noise-types']:

test_likelihoods_savedir = os.path.join(
cfg['git-results-basedir'], noise_type, data_split, 'imputation_benchmark'
)

all_covs_models_test_likelihoods_mean_df = pd.DataFrame(
np.nan,
index=cfg['plot-covs-types'],
Expand All @@ -120,8 +125,8 @@ def _plot_all_covs_structures_bar(
likelihoods_df = pd.read_csv(
test_likelihoods_savepath,
index_col=0,
) # (n_trials, n_all_covs_types)
likelihoods_df = likelihoods_df.loc[:, cfg['plot-covs-types']] # (n_trials, n_covs_types)
) # (num_trials, num_all_covs_types)
likelihoods_df = likelihoods_df.loc[:, cfg['plot-covs-types']] # (num_trials, num_covs_types)

# Update covs types labels for plots.
likelihoods_df.columns = likelihoods_df.columns.str.replace('periodic_1', 'periodic (slow)')
Expand Down Expand Up @@ -172,12 +177,11 @@ def _plot_all_covs_structures_bar(
)
for model_name in cfg['plot-models']:
likelihoods_filename = f'{data_split:s}_{noise_type:s}_likelihoods_{model_name:s}.csv'
test_likelihoods_savedir = os.path.join(cfg['git-results-basedir'], 'imputation_benchmark')
test_likelihoods_savepath = os.path.join(test_likelihoods_savedir, likelihoods_filename)
if os.path.exists(test_likelihoods_savepath):
likelihoods_df = pd.read_csv(
test_likelihoods_savepath, index_col=0
) # (n_trials, n_train_covs_types)
) # (num_trials, num_train_covs_types)
if not likelihoods_df.shape == (num_trials, len(cfg['all-covs-types'])):
logging.warning("Unexpected number of covariance structures found.")
print(likelihoods_df.round(3))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,13 @@
)
models_list = cfg['plot-models']
num_trials = int(experiment_data[-4:])
test_likelihoods_savedir = os.path.join(cfg['git-results-basedir'], 'imputation_benchmark')

for noise_type in cfg['noise-types']:

test_likelihoods_savedir = os.path.join(
cfg['git-results-basedir'], noise_type, data_split, 'imputation_benchmark'
)

for covs_type in cfg['plot-covs-types']:

all_method_significances_df = pd.DataFrame(
Expand Down

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

Loading

0 comments on commit af7ad30

Please sign in to comment.