From 0e29ba53ff3d490b43bd7eb5ad0ea643684d7e9f Mon Sep 17 00:00:00 2001 From: Jeroen Galjaard Date: Fri, 29 Apr 2022 13:02:58 +0200 Subject: [PATCH] Introduce initial changes to change defautl logging directory --- .../example_arrival_config.json | 10 +++---- experiments/node.jinja.yaml | 1 + fltk/core/distributed/orchestrator.py | 26 ++++++++++++++++--- fltk/util/config/config.py | 3 +-- 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/configs/federated_tasks/example_arrival_config.json b/configs/federated_tasks/example_arrival_config.json index ee7f06ae..8aa8202b 100644 --- a/configs/federated_tasks/example_arrival_config.json +++ b/configs/federated_tasks/example_arrival_config.json @@ -14,8 +14,8 @@ "memory": "1Gi" }, "Worker": { - "cores": "750m", - "memory": "1Gi" + "cores": "2000m", + "memory": "2Gi" } } }, @@ -64,11 +64,7 @@ }, "experimentConfiguration": { "randomSeed": [ - 1, - 41, - 42, - 43, - 430 + 1 ], "workerReplication": { "Master": 1, diff --git a/experiments/node.jinja.yaml b/experiments/node.jinja.yaml index fa1ace8e..122b806a 100644 --- a/experiments/node.jinja.yaml +++ b/experiments/node.jinja.yaml @@ -21,3 +21,4 @@ data_sampler_args: {{ task.get_sampler_args(tpe) }} replication_id: {{ task.get_net_param('replication') }} real_time: true save_data_append: true +output_path: {{ experiment_path }} \ No newline at end of file diff --git a/fltk/core/distributed/orchestrator.py b/fltk/core/distributed/orchestrator.py index ec476d5a..7a297aee 100644 --- a/fltk/core/distributed/orchestrator.py +++ b/fltk/core/distributed/orchestrator.py @@ -21,7 +21,25 @@ __ENV = Environment(loader=FileSystemLoader(EXPERIMENT_DIR)) -def _prepare_experiment_maps(task: FederatedArrivalTask, u_id, replication: int = 1) -> \ +def _generate_experiment_path_name(task: ArrivalTask, u_id: str, config: DistributedConfig): + """ + Helper function to generate experiment name for logging without conflicts + @param task: Arrival task for Task realted information. + @type task: ArrivalTask + @param u_id: Unique identifier string corresponding to the experiment. + @type u_id: str + @param config: Distributed configuration for logging directory configuration. + @type config: DistributedConfig + @return: String representation of the logging path for a specific experiment. + @rtype: str + """ + log_dir = config.execution_config.log_path + experiment_name = f"{task.dataset}_{task.network}_{u_id}_{task.replication}" + full_path = f"{log_dir}/{experiment_name}" + return full_path + + +def _prepare_experiment_maps(task: FederatedArrivalTask, config: DistributedConfig, u_id: str, replication: int = 1) -> \ (OrderedDict[str, V1ConfigMap], OrderedDict[str, str]): template = __ENV.get_template('node.jinja.yaml') type_dict = collections.OrderedDict() @@ -30,8 +48,8 @@ def _prepare_experiment_maps(task: FederatedArrivalTask, u_id, replication: int name = str(f'{tpe}-{u_id}-{replication}').lower() meta = V1ObjectMeta(name=name, labels={'app.kubernetes.io/name': f"fltk.node.config.{tpe}"}) - # TODO: Replication / seed information - filled_template = template.render(task=task, tpe=tpe, replication=replication) + exp_path = _generate_experiment_path_name(task, u_id, config) + filled_template = template.render(task=task, tpe=tpe, replication=replication, experiment_path=exp_path) type_dict[tpe] = V1ConfigMap(data={'node.config.yaml': filled_template}, metadata=meta) name_dict[tpe] = name return type_dict, name_dict @@ -171,7 +189,7 @@ def run_federated(self, clear: bool = True) -> None: # Do blocking request to priority queue curr_task = self.pending_tasks.get() self.__logger.info(f"Scheduling arrival of Arrival: {curr_task.id}") - config_dict, configmap_name_dict = _prepare_experiment_maps(curr_task, curr_task.id, 1) + config_dict, configmap_name_dict = _prepare_experiment_maps(curr_task, self._config, curr_task.id, 1) job_to_start = construct_job(self._config, curr_task, configmap_name_dict) self.__create_config_maps(config_dict) diff --git a/fltk/util/config/config.py b/fltk/util/config/config.py index f4e6e738..e0c52132 100644 --- a/fltk/util/config/config.py +++ b/fltk/util/config/config.py @@ -73,7 +73,7 @@ class Config: # Save data in append mode. Thereby flushing on every append to file. # This could be useful when a system is likely to crash midway an experiment save_data_append: bool = False - output_path: Path = Path('output_test_2') + output_path: Path = Path('logging') def __init__(self, **kwargs) -> None: enum_fields = [x for x in self.__dataclass_fields__.items() if isinstance(x[1].type, Enum) or isinstance(x[1].type, EnumMeta)] @@ -90,7 +90,6 @@ def __init__(self, **kwargs) -> None: self.output_path = Path(value) self.update_rng_seed() - def update_rng_seed(self): torch.manual_seed(self.rng_seed)