End to end testing: Fixture moved to a separate file (#1208)

* Use large runners Signed-off-by: noopur <[email protected]> * Use large runners Signed-off-by: noopur <[email protected]> * Use large runners Signed-off-by: noopur <[email protected]> * Corrected memory logs path Signed-off-by: noopur <[email protected]> * Moved fixture to a separate file Signed-off-by: noopur <[email protected]> * Print github contexts Signed-off-by: noopur <[email protected]> * Print github contexts Signed-off-by: noopur <[email protected]> * Print github contexts Signed-off-by: noopur <[email protected]> * Final changes Signed-off-by: noopur <[email protected]> * Comment for openfl image step Signed-off-by: noopur <[email protected]> --------- Signed-off-by: noopur <[email protected]>
securefederatedai · Dec 12, 2024 · 106fffc · 106fffc
1 parent 8ab9e3c
commit 106fffc
Show file tree

Hide file tree

Showing 10 changed files with 187 additions and 125 deletions.
diff --git a/.github/workflows/task_runner_docker_e2e.yml b/.github/workflows/task_runner_docker_e2e.yml
@@ -66,6 +66,14 @@ jobs:
           pip install .
           pip install -r test-requirements.txt
 
+      - name: Create OpenFL image
+        id: create_openfl_image
+        run: |
+          echo "Creating openfl image with current repo and branch. This may take a few minutes..."
+          cd openfl-docker
+          docker build . -t openfl -f Dockerfile.base \
+            --build-arg OPENFL_REVISION=https://github.com/${{ github.repository }}.git@${{ github.ref_name }}
+
       - name: Run Task Runner E2E tests with TLS
         id: run_tests
         run: |
@@ -134,6 +142,14 @@ jobs:
           pip install .
           pip install -r test-requirements.txt
 
+      - name: Create OpenFL image
+        id: create_openfl_image
+        run: |
+          echo "Creating openfl image with current repo and branch. This may take a few minutes..."
+          cd openfl-docker
+          docker build . -t openfl -f Dockerfile.base \
+            --build-arg OPENFL_REVISION=https://github.com/${{ github.repository }}.git@${{ github.ref_name }}
+
       - name: Run Task Runner E2E tests without TLS
         id: run_tests
         run: |
@@ -202,7 +218,15 @@ jobs:
           pip install .
           pip install -r test-requirements.txt
 
-      - name: Run Task Runner E2E tests without TLS
+      - name: Create OpenFL image
+        id: create_openfl_image
+        run: |
+          echo "Creating openfl image with current repo and branch. This may take a few minutes..."
+          cd openfl-docker
+          docker build . -t openfl -f Dockerfile.base \
+            --build-arg OPENFL_REVISION=https://github.com/${{ github.repository }}.git@${{ github.ref_name }}
+
+      - name: Run Task Runner E2E tests without Client Auth
         id: run_tests
         run: |
           python -m pytest -s tests/end_to_end/test_suites/docker_tests.py \

diff --git a/.github/workflows/task_runner_e2e.yml b/.github/workflows/task_runner_e2e.yml
@@ -272,7 +272,7 @@ jobs:
           pip install .
           pip install -r test-requirements.txt
 
-      - name: Run Task Runner E2E tests without TLS
+      - name: Run Task Runner E2E tests with TLS and memory logs
         id: run_tests
         run: |
           python -m pytest -s tests/end_to_end/test_suites/memory_logs_tests.py \
@@ -291,7 +291,7 @@ jobs:
       - name: Tar files
         id: tar_files
         if: ${{ always() }}
-        run: tar -cvf result.tar results
+        run: tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results
 
       - name: Upload Artifacts
         id: upload_artifacts

diff --git a/tests/end_to_end/README.md b/tests/end_to_end/README.md
@@ -8,7 +8,7 @@ This project aims at integration testing of ```openfl-workspace``` using pytest
 tests/end_to_end
 ├── models                  # Central location for all model-related code for testing purpose
 ├── test_suites             # Folder containing test files
-├── utils                   # Folder containing helper files
+├── utils                   # Folder containing fixture and helper files
 ├── conftest.py             # Pytest framework configuration file
 ├── pytest.ini              # Pytest initialisation file
 └── README.md               # Readme file

diff --git a/tests/end_to_end/conftest.py b/tests/end_to_end/conftest.py
@@ -2,25 +2,17 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import pytest
-import collections
+
 import os
 import shutil
 import xml.etree.ElementTree as ET
 import logging
-import concurrent.futures
+
 
 from tests.end_to_end.utils.logger import configure_logging
 from tests.end_to_end.utils.logger import logger as log
 from tests.end_to_end.utils.conftest_helper import parse_arguments
-import tests.end_to_end.utils.docker_helper as dh
-import tests.end_to_end.utils.federation_helper as fh
-from tests.end_to_end.models import aggregator as agg_model, model_owner as mo_model
 
-# Define a named tuple to store the objects for model owner, aggregator, and collaborators
-federation_fixture = collections.namedtuple(
-    "federation_fixture",
-    "model_owner, aggregator, collaborators, workspace_path, local_bind_path",
-)
 
 def pytest_addoption(parser):
     """
@@ -217,100 +209,3 @@ def pytest_configure(config):
     config.use_tls = not args.disable_tls
     config.log_memory_usage = args.log_memory_usage
     config.results_dir = config.getini("results_dir")
-
-
-@pytest.fixture(scope="function")
-def fx_federation(request):
-    """
-    Fixture for federation. This fixture is used to create the model owner, aggregator, and collaborators.
-    It also creates workspace.
-    Assumption: OpenFL workspace is present for the model being tested.
-    Args:
-        request: pytest request object. Model name is passed as a parameter to the fixture from test cases.
-    Returns:
-        federation_fixture: Named tuple containing the objects for model owner, aggregator, and collaborators
-
-    Note: As this is a function level fixture, thus no import is required at test level.
-    """
-    collaborators = []
-    executor = concurrent.futures.ThreadPoolExecutor()
-
-    test_env, model_name, workspace_path, local_bind_path, agg_domain_name = fh.federation_env_setup_and_validate(request)
-    agg_workspace_path = os.path.join(workspace_path, "aggregator", "workspace")
-
-    # Create model owner object and the workspace for the model
-    # Workspace name will be same as the model name
-    model_owner = mo_model.ModelOwner(model_name, request.config.log_memory_usage, workspace_path=agg_workspace_path)
-
-    # Create workspace for given model name
-    fh.create_persistent_store(model_owner.name, local_bind_path)
-
-    # Start the docker container for aggregator in case of docker environment
-    if test_env == "docker":
-        container = dh.start_docker_container(
-            container_name="aggregator",
-            workspace_path=workspace_path,
-            local_bind_path=local_bind_path,
-        )
-        model_owner.container_id = container.id
-
-    model_owner.create_workspace()
-    fh.add_local_workspace_permission(local_bind_path)
-
-    # Modify the plan
-    plan_path = os.path.join(local_bind_path, "aggregator", "workspace", "plan")
-    model_owner.modify_plan(
-        plan_path=plan_path,
-        new_rounds=request.config.num_rounds,
-        num_collaborators=request.config.num_collaborators,
-        disable_client_auth=not request.config.require_client_auth,
-        disable_tls=not request.config.use_tls,
-    )
-
-    # Certify the workspace in case of TLS
-    # Register the collaborators in case of non-TLS
-    if request.config.use_tls:
-        model_owner.certify_workspace()
-    else:
-        model_owner.register_collaborators(plan_path, request.config.num_collaborators)
-
-    # Initialize the plan
-    model_owner.initialize_plan(agg_domain_name=agg_domain_name)
-
-    # Create the objects for aggregator and collaborators
-    # Workspace path for aggregator is uniform in case of docker or task_runner
-    # But, for collaborators, it is different
-    aggregator = agg_model.Aggregator(
-        agg_domain_name=agg_domain_name,
-        workspace_path=agg_workspace_path,
-        container_id=model_owner.container_id, # None in case of non-docker environment
-    )
-
-    # Generate the sign request and certify the aggregator in case of TLS
-    if request.config.use_tls:
-        aggregator.generate_sign_request()
-        model_owner.certify_aggregator(agg_domain_name)
-
-    # Export the workspace
-    # By default the workspace will be exported to workspace.zip
-    model_owner.export_workspace()
-
-    futures = [
-        executor.submit(
-            fh.setup_collaborator,
-            count=i,
-            workspace_path=workspace_path,
-            local_bind_path=local_bind_path,
-        )
-        for i in range(request.config.num_collaborators)
-    ]
-    collaborators = [f.result() for f in futures]
-
-    # Return the federation fixture
-    return federation_fixture(
-        model_owner=model_owner,
-        aggregator=aggregator,
-        collaborators=collaborators,
-        workspace_path=workspace_path,
-        local_bind_path=local_bind_path,
-    )
diff --git a/tests/end_to_end/test_suites/docker_tests.py b/tests/end_to_end/test_suites/docker_tests.py
@@ -4,6 +4,7 @@
 import pytest
 import logging
 
+from tests.end_to_end.utils.common_fixtures import fx_federation
 from tests.end_to_end.utils import federation_helper as fed_helper
 
 log = logging.getLogger(__name__)

diff --git a/tests/end_to_end/test_suites/memory_logs_tests.py b/tests/end_to_end/test_suites/memory_logs_tests.py
@@ -6,6 +6,7 @@
 import os
 import json
 
+from tests.end_to_end.utils.common_fixtures import fx_federation
 from tests.end_to_end.utils import federation_helper as fed_helper
 
 log = logging.getLogger(__name__)
@@ -38,36 +39,55 @@ def test_log_memory_usage(request, fx_federation):
 
     # Setup PKI for trusted communication within the federation
     if request.config.use_tls:
-        assert fed_helper.setup_pki(fx_federation), "Failed to setup PKI for trusted communication"
+        assert fed_helper.setup_pki(
+            fx_federation
+        ), "Failed to setup PKI for trusted communication"
 
     # Start the federation
     results = fed_helper.run_federation(fx_federation)
 
     # Verify the completion of the federation run
-    assert fed_helper.verify_federation_run_completion(fx_federation, results, \
-                                num_rounds=request.config.num_rounds), "Federation completion failed"
+    assert fed_helper.verify_federation_run_completion(
+        fx_federation, results, num_rounds=request.config.num_rounds
+    ), "Federation completion failed"
     # Verify the aggregator memory logs
-    aggregator_memory_usage_file = os.path.join(fx_federation.workspace_path, "logs", "aggregator_memory_usage.json")
-    assert os.path.exists(aggregator_memory_usage_file), "Aggregator memory usage file is not available"
+    aggregator_memory_usage_file = os.path.join(
+        fx_federation.workspace_path,
+        "aggregator",
+        "workspace",
+        "logs",
+        "aggregator_memory_usage.json",
+    )
+    assert os.path.exists(
+        aggregator_memory_usage_file
+    ), "Aggregator memory usage file is not available"
 
     # Log the aggregator memory usage details
     memory_usage_dict = json.load(open(aggregator_memory_usage_file))
 
     # check memory usage entries for each round
-    assert len(memory_usage_dict) == request.config.num_rounds, \
-                "Memory usage details are not available for all rounds"
+    assert (
+        len(memory_usage_dict) == request.config.num_rounds
+    ), "Memory usage details are not available for all rounds"
 
     # check memory usage entries for each collaborator
     for collaborator in fx_federation.collaborators:
-        collaborator_memory_usage_file = os.path.join(fx_federation.workspace_path,
-                                                      "logs",
-                                                      f"{collaborator.collaborator_name}_memory_usage.json")
+        collaborator_memory_usage_file = os.path.join(
+            fx_federation.workspace_path,
+            collaborator.name,
+            "workspace",
+            "logs",
+            f"{collaborator.collaborator_name}_memory_usage.json",
+        )
 
-        assert os.path.exists(collaborator_memory_usage_file), f"Memory usage file for collaborator {collaborator.collaborator_name} is not available"
+        assert os.path.exists(
+            collaborator_memory_usage_file
+        ), f"Memory usage file for collaborator {collaborator.collaborator_name} is not available"
 
         memory_usage_dict = json.load(open(collaborator_memory_usage_file))
 
-        assert len(memory_usage_dict) == request.config.num_rounds, \
-                f"Memory usage details are not available for all rounds for collaborator {collaborator.collaborator_name}"
+        assert (
+            len(memory_usage_dict) == request.config.num_rounds
+        ), f"Memory usage details are not available for all rounds for collaborator {collaborator.collaborator_name}"
 
     log.info("Memory usage details are available for all participants")
diff --git a/tests/end_to_end/test_suites/sample_tests.py b/tests/end_to_end/test_suites/sample_tests.py
@@ -4,6 +4,7 @@
 import pytest
 import logging
 
+from tests.end_to_end.utils.common_fixtures import fx_federation
 from tests.end_to_end.utils import federation_helper as fed_helper
 
 log = logging.getLogger(__name__)

diff --git a/tests/end_to_end/test_suites/task_runner_tests.py b/tests/end_to_end/test_suites/task_runner_tests.py
@@ -4,6 +4,7 @@
 import pytest
 import logging
 
+from tests.end_to_end.utils.common_fixtures import fx_federation
 from tests.end_to_end.utils import federation_helper as fed_helper
 
 log = logging.getLogger(__name__)