- federated evaluation with defautls baked in aggregator, tasks and a…

…ssigner (#1186) - updated torch_cnn_mnist_fed_eval sample workspace to use new defaults - Update about/feature documentation to reflect these changes Signed-off-by: Shailesh Pant <[email protected]>
securefederatedai · Dec 3, 2024 · 7c53fb7 · 7c53fb7
1 parent fa59d3f
commit 7c53fb7
Show file tree

Hide file tree

Showing 5 changed files with 33 additions and 24 deletions.
diff --git a/docs/about/features_index/fed_eval.rst b/docs/about/features_index/fed_eval.rst
@@ -32,11 +32,17 @@ This script can be directly executed as follows:
 
     $ python test_hello_federation.py --template torch_cnn_mnist_fed_eval
     
-In order to adapt this template for federated evaluation, the following modifications were made to ``plan.yaml``:
+In order to adapt this template for federated evaluation, the following defaults were added for assigner, aggregator and tasks and same referenced in the ``plan.yaml``:
 
 .. literalinclude:: ../../../openfl-workspace/torch_cnn_mnist_fed_eval/plan/plan.yaml
 
-Key Changes for Federated Evaluation:
+.. literalinclude:: ../../../openfl-workspace/workspace/plan/defaults/federated-evaluation/aggregator.yaml
+
+.. literalinclude:: ../../../openfl-workspace/workspace/plan/defaults/federated-evaluation/assigner.yaml
+
+.. literalinclude:: ../../../openfl-workspace/workspace/plan/defaults/federated-evaluation/tasks_torch.yaml
+
+Key Changes for Federated Evaluation by baking in defaults for:
 
 1. **aggregator.settings.rounds_to_train**: Set to 1
 2. **assigner**: Assign to aggregated_model_validation instead of default assignments
@@ -48,4 +54,4 @@ This sample script will create a federation based on the `torch_cnn_mnist_fed_ev
 
 ---
 
-Congratulations, you have successfully performed federated evaluation across two decentralized collaborator nodes.
+Congratulations, you have successfully performed federated evaluation across two decentralized collaborator nodes with minor default reference changes to plan
diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/plan/plan.yaml b/openfl-workspace/torch_cnn_mnist_fed_eval/plan/plan.yaml
@@ -2,15 +2,12 @@
 # Licensed subject to the terms of the separately executed evaluation license agreement between Intel Corporation and you.
 
 aggregator :
-  defaults : plan/defaults/aggregator.yaml
+  defaults : plan/defaults/federated-evaluation/aggregator.yaml
   template : openfl.component.Aggregator
   settings :
     init_state_path     : save/torch_cnn_mnist_init.pbuf
     best_state_path     : save/torch_cnn_mnist_best.pbuf
     last_state_path     : save/torch_cnn_mnist_last.pbuf
-    ######### SET ROUNDS TO 1 #############
-    rounds_to_train     : 1
-    #######################################
     log_metric_callback :
       template : src.mnist_utils.write_metric
 
@@ -37,25 +34,10 @@ network :
   defaults : plan/defaults/network.yaml
 
 assigner :
-  ######### SET ASSIGNER TO ONLY INCLUDE AGGREGATED MODEL VALIDATION #############
-  template : openfl.component.RandomGroupedAssigner
-  settings :
-    task_groups  :
-      - name       : validate
-        percentage : 1.0
-        tasks      :
-          - aggregated_model_validation
-  ################################################################################
+  defaults : plan/defaults/federated-evaluation/assigner.yaml
 
 tasks :
-  ######### SET AGGREGATED MODEL VALIDATION AS ONLY TASK #############
-  aggregated_model_validation:
-    function : validate
-    kwargs   :
-      apply   : global
-      metrics :
-        - acc
-  ####################################################################
+  defaults : plan/defaults/federated-evaluation/tasks_torch.yaml
 
 compression_pipeline :
   defaults : plan/defaults/compression_pipeline.yaml
diff --git a/openfl-workspace/workspace/plan/defaults/federated-evaluation/aggregator.yaml b/openfl-workspace/workspace/plan/defaults/federated-evaluation/aggregator.yaml
@@ -0,0 +1,7 @@
+template : openfl.component.Aggregator
+settings :
+    db_store_rounds : 2
+    write_logs      : true
+    ######### Default rounds to 1 for evaluation #############
+    rounds_to_train     : 1
+    ##########################################################
diff --git a/openfl-workspace/workspace/plan/defaults/federated-evaluation/assigner.yaml b/openfl-workspace/workspace/plan/defaults/federated-evaluation/assigner.yaml
@@ -0,0 +1,7 @@
+template : openfl.component.RandomGroupedAssigner
+settings :
+  task_groups  :
+    - name       : validate
+      percentage : 1.0
+      tasks      :
+        - aggregated_model_validation
diff --git a/openfl-workspace/workspace/plan/defaults/federated-evaluation/tasks_torch.yaml b/openfl-workspace/workspace/plan/defaults/federated-evaluation/tasks_torch.yaml
@@ -0,0 +1,7 @@
+aggregated_model_validation:
+  function : validate_task
+  kwargs   :
+    apply   : global
+    metrics :
+      - acc
+