From 7c53fb75a4ab22465c003eb06ad9862b0b76d5c8 Mon Sep 17 00:00:00 2001 From: ishaileshpant Date: Tue, 3 Dec 2024 21:22:43 +0530 Subject: [PATCH] - federated evaluation with defautls baked in aggregator, tasks and assigner (#1186) - updated torch_cnn_mnist_fed_eval sample workspace to use new defaults - Update about/feature documentation to reflect these changes Signed-off-by: Shailesh Pant --- docs/about/features_index/fed_eval.rst | 12 +++++++--- .../torch_cnn_mnist_fed_eval/plan/plan.yaml | 24 +++---------------- .../federated-evaluation/aggregator.yaml | 7 ++++++ .../federated-evaluation/assigner.yaml | 7 ++++++ .../federated-evaluation/tasks_torch.yaml | 7 ++++++ 5 files changed, 33 insertions(+), 24 deletions(-) create mode 100644 openfl-workspace/workspace/plan/defaults/federated-evaluation/aggregator.yaml create mode 100644 openfl-workspace/workspace/plan/defaults/federated-evaluation/assigner.yaml create mode 100644 openfl-workspace/workspace/plan/defaults/federated-evaluation/tasks_torch.yaml diff --git a/docs/about/features_index/fed_eval.rst b/docs/about/features_index/fed_eval.rst index af4aa39092..e8ec5fbdd7 100644 --- a/docs/about/features_index/fed_eval.rst +++ b/docs/about/features_index/fed_eval.rst @@ -32,11 +32,17 @@ This script can be directly executed as follows: $ python test_hello_federation.py --template torch_cnn_mnist_fed_eval -In order to adapt this template for federated evaluation, the following modifications were made to ``plan.yaml``: +In order to adapt this template for federated evaluation, the following defaults were added for assigner, aggregator and tasks and same referenced in the ``plan.yaml``: .. literalinclude:: ../../../openfl-workspace/torch_cnn_mnist_fed_eval/plan/plan.yaml -Key Changes for Federated Evaluation: +.. literalinclude:: ../../../openfl-workspace/workspace/plan/defaults/federated-evaluation/aggregator.yaml + +.. literalinclude:: ../../../openfl-workspace/workspace/plan/defaults/federated-evaluation/assigner.yaml + +.. literalinclude:: ../../../openfl-workspace/workspace/plan/defaults/federated-evaluation/tasks_torch.yaml + +Key Changes for Federated Evaluation by baking in defaults for: 1. **aggregator.settings.rounds_to_train**: Set to 1 2. **assigner**: Assign to aggregated_model_validation instead of default assignments @@ -48,4 +54,4 @@ This sample script will create a federation based on the `torch_cnn_mnist_fed_ev --- -Congratulations, you have successfully performed federated evaluation across two decentralized collaborator nodes. \ No newline at end of file +Congratulations, you have successfully performed federated evaluation across two decentralized collaborator nodes with minor default reference changes to plan \ No newline at end of file diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/plan/plan.yaml b/openfl-workspace/torch_cnn_mnist_fed_eval/plan/plan.yaml index 2eca7b67c4..02db7dff3e 100644 --- a/openfl-workspace/torch_cnn_mnist_fed_eval/plan/plan.yaml +++ b/openfl-workspace/torch_cnn_mnist_fed_eval/plan/plan.yaml @@ -2,15 +2,12 @@ # Licensed subject to the terms of the separately executed evaluation license agreement between Intel Corporation and you. aggregator : - defaults : plan/defaults/aggregator.yaml + defaults : plan/defaults/federated-evaluation/aggregator.yaml template : openfl.component.Aggregator settings : init_state_path : save/torch_cnn_mnist_init.pbuf best_state_path : save/torch_cnn_mnist_best.pbuf last_state_path : save/torch_cnn_mnist_last.pbuf - ######### SET ROUNDS TO 1 ############# - rounds_to_train : 1 - ####################################### log_metric_callback : template : src.mnist_utils.write_metric @@ -37,25 +34,10 @@ network : defaults : plan/defaults/network.yaml assigner : - ######### SET ASSIGNER TO ONLY INCLUDE AGGREGATED MODEL VALIDATION ############# - template : openfl.component.RandomGroupedAssigner - settings : - task_groups : - - name : validate - percentage : 1.0 - tasks : - - aggregated_model_validation - ################################################################################ + defaults : plan/defaults/federated-evaluation/assigner.yaml tasks : - ######### SET AGGREGATED MODEL VALIDATION AS ONLY TASK ############# - aggregated_model_validation: - function : validate - kwargs : - apply : global - metrics : - - acc - #################################################################### + defaults : plan/defaults/federated-evaluation/tasks_torch.yaml compression_pipeline : defaults : plan/defaults/compression_pipeline.yaml diff --git a/openfl-workspace/workspace/plan/defaults/federated-evaluation/aggregator.yaml b/openfl-workspace/workspace/plan/defaults/federated-evaluation/aggregator.yaml new file mode 100644 index 0000000000..df6fbcfdd6 --- /dev/null +++ b/openfl-workspace/workspace/plan/defaults/federated-evaluation/aggregator.yaml @@ -0,0 +1,7 @@ +template : openfl.component.Aggregator +settings : + db_store_rounds : 2 + write_logs : true + ######### Default rounds to 1 for evaluation ############# + rounds_to_train : 1 + ########################################################## diff --git a/openfl-workspace/workspace/plan/defaults/federated-evaluation/assigner.yaml b/openfl-workspace/workspace/plan/defaults/federated-evaluation/assigner.yaml new file mode 100644 index 0000000000..9d583fa0c4 --- /dev/null +++ b/openfl-workspace/workspace/plan/defaults/federated-evaluation/assigner.yaml @@ -0,0 +1,7 @@ +template : openfl.component.RandomGroupedAssigner +settings : + task_groups : + - name : validate + percentage : 1.0 + tasks : + - aggregated_model_validation \ No newline at end of file diff --git a/openfl-workspace/workspace/plan/defaults/federated-evaluation/tasks_torch.yaml b/openfl-workspace/workspace/plan/defaults/federated-evaluation/tasks_torch.yaml new file mode 100644 index 0000000000..f497ca845c --- /dev/null +++ b/openfl-workspace/workspace/plan/defaults/federated-evaluation/tasks_torch.yaml @@ -0,0 +1,7 @@ +aggregated_model_validation: + function : validate_task + kwargs : + apply : global + metrics : + - acc + \ No newline at end of file