Skip to content

Commit

Permalink
Update hyperparameter_importance.py
Browse files Browse the repository at this point in the history
  • Loading branch information
@janvanrijn
janvanrijn authored Oct 15, 2024
1 parent b9b1313 commit 8dc2efb
Showing 1 changed file with 45 additions and 38 deletions.
83 changes: 45 additions & 38 deletions publications/2023-neurips/experiments/surf/snellius/analysis/hyperparameter_importance.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,13 @@
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('--n_trees', type=int, default=16)
parser.add_argument('--openml_ids', type=int, nargs='+', default=[3])
parser.add_argument('--openml_ids', type=int, nargs='+', default=[3, 6])
parser.add_argument('--workflow_name', type=str, default="lcdb.workflow.sklearn.LibLinearWorkflow")
parser.add_argument('--openml_taskid_name', type=str, default="m:openmlid")
parser.add_argument('--output_directory', type=str, default=os.path.expanduser('~/experiments/lcdb'))
parser.add_argument('--output_filetype', type=str, choices=['pdf', 'png'], default='png')
parser.add_argument('--max_load', type=int, default=None)
parser.add_argument('--anchor_value', type=int, default=2048)
parser.add_argument('--anchor_values', type=int, nargs='+', default=[128, 512, 2048, -1])
return parser.parse_args()


Expand All @@ -44,7 +44,7 @@ def numeric_encode(df, config_space):
return result


def fanova_on_task(task_results, performance_column_name, config_space, n_trees):
def fanova_on_task(task_results, performance_column_name, current_anchor_value, config_space, n_trees):
fanova_results = []

evaluator = fanova.fanova.fANOVA(
Expand All @@ -61,14 +61,15 @@ def fanova_on_task(task_results, performance_column_name, config_space, n_trees)

fanova_results.append({
"hyperparameter": pname,
"fanova": importance[(idx,)]["individual importance"],
"anchor": current_anchor_value,
"variance_contribution": importance[(idx,)]["individual importance"],
})
return fanova_results


def run(args):
fanova_all_results = []
performance_column = "objective"
performance_column = "final_objective" # make sure to give this a unique name (not same as the "objective" field)
anchor_size_column = "anchor_sizes"
learning_curve_column = "learning_curve_data"

Expand All @@ -95,50 +96,54 @@ def run(args):
# job_ids = frame_workflow_job_task['job_id'].unique()
if len(workflow_ids) > 1 or len(openml_task_ids) > 1:
raise ValueError('Should not happen. %s %s' % (str(workflow_ids), str(openml_task_ids)))
if (workflow_ids[0], openml_task_ids[0]) not in id_results:
id_results[(workflow_ids[0], openml_task_ids[0])] = list()

performance_values_new = list()
for index, row in frame_workflow_job_task.iterrows():
anchor_sizes = row[anchor_size_column]
performance_value_at_anchor = np.nan
if args.anchor_value is not None:
if args.anchor_value not in anchor_sizes:
logging.warning('Anchor %d not available in task %d workflow %s'
% (args.anchor_value, openml_task_ids[0], workflow_ids[0])
)

for current_anchor_value in args.anchor_values:
if (workflow_ids[0], openml_task_ids[0], current_anchor_value) not in id_results:
id_results[(workflow_ids[0], openml_task_ids[0], current_anchor_value)] = list()

performance_values_new = list()
for index, row in frame_workflow_job_task.iterrows():
anchor_sizes = row[anchor_size_column]
performance_value_at_anchor = np.nan
if current_anchor_value != -1:
if current_anchor_value not in anchor_sizes:
logging.warning('Anchor %d not available in task %d workflow %s'
% (current_anchor_value, openml_task_ids[0], workflow_ids[0])
)
else:
anchor_index = anchor_sizes.index(current_anchor_value)
performance_value_at_anchor = row[learning_curve_column][anchor_index]
else:
anchor_index = anchor_sizes.index(args.anchor_value)
performance_value_at_anchor = row[learning_curve_column][anchor_index]
else:
performance_value_at_anchor = row[learning_curve_column][-1]
performance_values_new.append(performance_value_at_anchor)
performance_values_new = np.array(performance_values_new, dtype=float)
frame_workflow_job_task[performance_column] = pd.Series(performance_values_new)
performance_value_at_anchor = row[learning_curve_column][-1]
performance_values_new.append(performance_value_at_anchor)
performance_values_new = np.array(performance_values_new, dtype=float)
frame_workflow_job_task[performance_column] = pd.Series(performance_values_new)

id_results[(workflow_ids[0], openml_task_ids[0])].append(frame_workflow_job_task)
id_results[(workflow_ids[0], openml_task_ids[0], current_anchor_value)].append(frame_workflow_job_task)

load_count += 1
if args.max_load and load_count >= args.max_load:
break
load_count += 1
if args.max_load and load_count >= args.max_load:
break

task_ids = set()
for idx, (workflow_name, task_id) in enumerate(id_results):
for idx, (workflow_name, task_id, current_anchor_value) in enumerate(id_results):
task_ids.add(task_id)
task_results = pd.concat(id_results[(workflow_name, task_id)])
task_results = pd.concat(id_results[(workflow_name, task_id, current_anchor_value)])
task_results = task_results.rename(workflow_hyperparameter_mapping, axis=1)
relevant_columns = list(workflow_hyperparameter_mapping.values()) + [performance_column]
task_results = task_results[relevant_columns]

logging.info("Starting with task %d (%d/%d)" % (task_id, idx + 1, len(id_results)))
fanova_task_results = fanova_on_task(task_results, performance_column, config_space, args.n_trees)
logging.info("Starting with task %d anchor %d (%d/%d)" % (task_id, current_anchor_value, idx + 1, len(id_results)))
fanova_task_results = fanova_on_task(
task_results, performance_column, current_anchor_value, config_space, args.n_trees
)
fanova_all_results.extend(fanova_task_results)

fanova_all_results = pd.DataFrame(fanova_all_results)

# generate plot
fig, ax = plt.subplots(figsize=(16, 9))
sns.boxplot(x="hyperparameter", y="fanova", data=fanova_all_results, ax=ax)
sns.boxplot(x="hyperparameter", y="variance_contribution", hue="anchor", data=fanova_all_results, ax=ax)
ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha="right")
ax.set_ylabel("Variance Contribution")
ax.set_xlabel(None)
Expand All @@ -147,12 +152,14 @@ def run(args):

# save plot to file
filename_suffix = ""
if args.anchor_value is not None:
filename_suffix = "_anchor_%d" % args.anchor_value
output_file = args.output_directory + '/fanova_%s%s.%s' % (args.workflow_name, filename_suffix, args.output_filetype)
if args.anchor_values is not None:
filename_suffix = "_anchor_%s" % str(args.anchor_values)
output_file_base = args.output_directory + '/fanova_%s%s' % (args.workflow_name, filename_suffix)
os.makedirs(args.output_directory, exist_ok=True)
plt.savefig(output_file)
logging.info('saved to %s' % output_file)
fanova_all_results.to_csv(output_file_base + '.csv')
plt.savefig(output_file_base + '.' + args.output_filetype)
logging.info('saved plot to %s.%s' % (output_file_base, args.output_filetype))
logging.info('saved csv to %s.csv' % output_file_base)


if __name__ == '__main__':
Expand Down

0 comments on commit 8dc2efb

Please sign in to comment.