From c4a0206d3426aeefdcf02976fb0754c54d1fca09 Mon Sep 17 00:00:00 2001 From: Shailesh Pant Date: Mon, 16 Dec 2024 22:34:11 +0530 Subject: [PATCH] - implement a new ModeBasedAssigner class extending RandomGroupedAssigner - update defaults/assigner.yaml to use ModeBasedAssigner with default mode to train_and_validate - update fedeval sample workspace to use default assigner, tasks and aggregator - use of federated-evaluation/aggregator.yaml for FedEval specific workspace example to use round_number as 1 - removed assigner and tasks yaml from defaults/federated-evaluation, superseded by default assigner/tasks - add happy flow / negative tests for ModeBasedAssigner - TODO revert workflow change post validation on workflow run in draft mode Signed-off-by: Shailesh Pant --- .github/workflows/pytest_coverage.yml | 2 +- .../torch_cnn_mnist_fed_eval/plan/plan.yaml | 8 +- .../workspace/plan/defaults/assigner.yaml | 7 +- .../federated-evaluation/assigner.yaml | 7 -- .../federated-evaluation/tasks_torch.yaml | 7 -- openfl/component/__init__.py | 1 + openfl/component/assigner/__init__.py | 1 + .../component/assigner/mode_based_assigner.py | 62 ++++++++++ .../assigner/test_mode_based_assigner.py | 113 ++++++++++++++++++ 9 files changed, 189 insertions(+), 19 deletions(-) delete mode 100644 openfl-workspace/workspace/plan/defaults/federated-evaluation/assigner.yaml delete mode 100644 openfl-workspace/workspace/plan/defaults/federated-evaluation/tasks_torch.yaml create mode 100644 openfl/component/assigner/mode_based_assigner.py create mode 100644 tests/openfl/component/assigner/test_mode_based_assigner.py diff --git a/.github/workflows/pytest_coverage.yml b/.github/workflows/pytest_coverage.yml index 9371f74e13..26e5d39c3c 100644 --- a/.github/workflows/pytest_coverage.yml +++ b/.github/workflows/pytest_coverage.yml @@ -18,7 +18,7 @@ env: jobs: build: - if: github.event.pull_request.draft == false + if: github.event.pull_request.draft == true runs-on: ubuntu-latest timeout-minutes: 15 diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/plan/plan.yaml b/openfl-workspace/torch_cnn_mnist_fed_eval/plan/plan.yaml index 02db7dff3e..d5f9a17b11 100644 --- a/openfl-workspace/torch_cnn_mnist_fed_eval/plan/plan.yaml +++ b/openfl-workspace/torch_cnn_mnist_fed_eval/plan/plan.yaml @@ -34,10 +34,12 @@ network : defaults : plan/defaults/network.yaml assigner : - defaults : plan/defaults/federated-evaluation/assigner.yaml - + defaults : plan/defaults/assigner.yaml + settings : + mode : evaluate + tasks : - defaults : plan/defaults/federated-evaluation/tasks_torch.yaml + defaults : plan/defaults/tasks_torch.yaml compression_pipeline : defaults : plan/defaults/compression_pipeline.yaml diff --git a/openfl-workspace/workspace/plan/defaults/assigner.yaml b/openfl-workspace/workspace/plan/defaults/assigner.yaml index 0b7e744475..fe4eb5890d 100644 --- a/openfl-workspace/workspace/plan/defaults/assigner.yaml +++ b/openfl-workspace/workspace/plan/defaults/assigner.yaml @@ -1,4 +1,4 @@ -template : openfl.component.RandomGroupedAssigner +template : openfl.component.ModeBasedAssigner settings : task_groups : - name : train_and_validate @@ -7,3 +7,8 @@ settings : - aggregated_model_validation - train - locally_tuned_model_validation + - name : evaluate + percentage : 1.0 + tasks : + - aggregated_model_validation + mode: train_and_validate diff --git a/openfl-workspace/workspace/plan/defaults/federated-evaluation/assigner.yaml b/openfl-workspace/workspace/plan/defaults/federated-evaluation/assigner.yaml deleted file mode 100644 index 9d583fa0c4..0000000000 --- a/openfl-workspace/workspace/plan/defaults/federated-evaluation/assigner.yaml +++ /dev/null @@ -1,7 +0,0 @@ -template : openfl.component.RandomGroupedAssigner -settings : - task_groups : - - name : validate - percentage : 1.0 - tasks : - - aggregated_model_validation \ No newline at end of file diff --git a/openfl-workspace/workspace/plan/defaults/federated-evaluation/tasks_torch.yaml b/openfl-workspace/workspace/plan/defaults/federated-evaluation/tasks_torch.yaml deleted file mode 100644 index f497ca845c..0000000000 --- a/openfl-workspace/workspace/plan/defaults/federated-evaluation/tasks_torch.yaml +++ /dev/null @@ -1,7 +0,0 @@ -aggregated_model_validation: - function : validate_task - kwargs : - apply : global - metrics : - - acc - \ No newline at end of file diff --git a/openfl/component/__init__.py b/openfl/component/__init__.py index 3b787f87d0..df90f1b428 100644 --- a/openfl/component/__init__.py +++ b/openfl/component/__init__.py @@ -5,6 +5,7 @@ from openfl.component.aggregator.aggregator import Aggregator from openfl.component.assigner.assigner import Assigner from openfl.component.assigner.random_grouped_assigner import RandomGroupedAssigner +from openfl.component.assigner.mode_based_assigner import ModeBasedAssigner from openfl.component.assigner.static_grouped_assigner import StaticGroupedAssigner from openfl.component.collaborator.collaborator import Collaborator from openfl.component.straggler_handling_functions.cutoff_time_based_straggler_handling import ( diff --git a/openfl/component/assigner/__init__.py b/openfl/component/assigner/__init__.py index 980a524b7f..316f9a8e2b 100644 --- a/openfl/component/assigner/__init__.py +++ b/openfl/component/assigner/__init__.py @@ -5,3 +5,4 @@ from openfl.component.assigner.assigner import Assigner from openfl.component.assigner.random_grouped_assigner import RandomGroupedAssigner from openfl.component.assigner.static_grouped_assigner import StaticGroupedAssigner +from openfl.component.assigner.mode_based_assigner import ModeBasedAssigner \ No newline at end of file diff --git a/openfl/component/assigner/mode_based_assigner.py b/openfl/component/assigner/mode_based_assigner.py new file mode 100644 index 0000000000..cb747cec0d --- /dev/null +++ b/openfl/component/assigner/mode_based_assigner.py @@ -0,0 +1,62 @@ +# Copyright 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + + +"""Mode based assigner module.""" + +from openfl.component.assigner.random_grouped_assigner import RandomGroupedAssigner + + +class ModeBasedAssigner(RandomGroupedAssigner): + r"""The task assigner maintains a list of tasks. + + Also it decides the policy for which collaborator should run those tasks + There may be many types of policies implemented, but a natural place to + start is with a: + + - ModeBasedAssigner : + Given a set of task groups and mode it filters the task groups based + on mode. It futher enforces checks for specific modes. + Post filtering it deletgates the task assignment to RandomGroupedAssigner. + + Attributes: + task_groups* (list of object): Task groups to assign. + mode* (str): Mode to determine task assignments. + + .. note:: + \* - Plan setting. + """ + + def __init__(self, task_groups, mode, **kwargs): + """Initializes the ModeBasedAssigner. + + Args: + task_groups (list of object): Task groups to assign. + mode (str): Mode to determine task assignments. + **kwargs: Additional keyword arguments. + """ + self.task_groups = task_groups + self.mode = mode + super().__init__(task_groups=task_groups, **kwargs) + + def define_task_assignments(self): + """Define task assignments for each round and collaborator. + + This method filters tasks for the + collaborators for each round based on the mode. + + Args: + None + + Returns: + None + """ + self.task_groups = [ + group for group in self.task_groups + if group["name"] == self.mode + ] + if self.mode == "evaluate" : + assert ( + self.rounds == 1 + ), "Number of rounds should be 1 for evaluate mode" + super().define_task_assignments() \ No newline at end of file diff --git a/tests/openfl/component/assigner/test_mode_based_assigner.py b/tests/openfl/component/assigner/test_mode_based_assigner.py new file mode 100644 index 0000000000..00204b728f --- /dev/null +++ b/tests/openfl/component/assigner/test_mode_based_assigner.py @@ -0,0 +1,113 @@ +import pytest +from openfl.component.assigner.mode_based_assigner import ModeBasedAssigner + +@pytest.fixture +def sample_tasks(): + return ["task1", "task2", "task3", "task4"] + +@pytest.fixture +def sample_authorized_cols(): + return ["col1", "col2", "col3"] + +@pytest.fixture +def sample_task_groups(): + return [ + {"name": "train", "tasks": ["task1", "task2"], "percentage": 1.0}, + {"name": "evaluate", "tasks": ["task3"], "percentage": 1.0}, + {"name": "random-mode", "tasks": ["task4"], "percentage": 1.0} + ] + +def test_init_with_valid_mode(sample_tasks, sample_authorized_cols): + """Test initialization with valid mode and task groups.""" + task_groups = [{"name": "train", "tasks": ["task1"], "percentage": 1.0}] + assigner = ModeBasedAssigner( + task_groups=task_groups, + mode="train", + tasks=sample_tasks, + authorized_cols=sample_authorized_cols, + rounds_to_train=3 + ) + assert assigner.mode == "train" + assert assigner.task_groups == task_groups + +def test_define_task_assignments_train_mode(sample_task_groups, sample_tasks, sample_authorized_cols): + """Test task assignments filtering for train mode.""" + assigner = ModeBasedAssigner( + task_groups=sample_task_groups, + mode="train", + tasks=sample_tasks, + authorized_cols=sample_authorized_cols, + rounds_to_train=3 + ) + assigner.define_task_assignments() + assert len(assigner.task_groups) == 1 + assert assigner.task_groups[0]["name"] == "train" + +def test_define_task_assignments_evaluate_mode(sample_task_groups, sample_tasks, sample_authorized_cols): + """Test task assignments filtering for evaluate mode with rounds=1.""" + assigner = ModeBasedAssigner( + task_groups=sample_task_groups, + mode="evaluate", + tasks=sample_tasks, + authorized_cols=sample_authorized_cols, + rounds_to_train=1 + ) + assigner.define_task_assignments() + assert len(assigner.task_groups) == 1 + assert assigner.task_groups[0]["name"] == "evaluate" + +def test_evaluate_mode_with_invalid_rounds(sample_task_groups, sample_tasks, sample_authorized_cols): + """Test that evaluate mode raises error when rounds > 1.""" + assigner = ModeBasedAssigner( + task_groups=sample_task_groups, + mode="evaluate", + tasks=sample_tasks, + authorized_cols=sample_authorized_cols, + rounds_to_train=2 + ) + with pytest.raises(AssertionError): + assigner.define_task_assignments() + +def test_empty_task_groups_after_filtering(sample_tasks, sample_authorized_cols): + """Test behavior when no task groups match the specified mode.""" + task_groups = [{"name": "train", "tasks": ["task1"], "percentage": 1}] + assigner = ModeBasedAssigner( + task_groups=task_groups, + mode="non_existent_mode", + tasks=sample_tasks, + authorized_cols=sample_authorized_cols, + rounds_to_train=1 + ) + with pytest.raises(AssertionError): + assigner.define_task_assignments() + +def test_multiple_matching_task_groups(sample_tasks, sample_authorized_cols): + """Test behavior when multiple task groups match the mode.""" + task_groups = [ + {"name": "train", "tasks": ["task1"], "percentage": 0.6}, + {"name": "train", "tasks": ["task2"], "percentage": 0.4} + ] + assigner = ModeBasedAssigner( + task_groups=task_groups, + mode="train", + tasks=sample_tasks, + authorized_cols=sample_authorized_cols, + rounds_to_train=1 + ) + assigner.define_task_assignments() + +def test_multiple_matching_tg_percentage_error(sample_tasks, sample_authorized_cols): + """Test behavior when multiple task groups match the mode.""" + task_groups = [ + {"name": "train", "tasks": ["task1"], "percentage": 1}, + {"name": "train", "tasks": ["task2"], "percentage": 1} + ] + assigner = ModeBasedAssigner( + task_groups=task_groups, + mode="train", + tasks=sample_tasks, + authorized_cols=sample_authorized_cols, + rounds_to_train=1 + ) + with pytest.raises(AssertionError): + assigner.define_task_assignments()