diff --git a/.github/workflows/task_runner_docker_e2e.yml b/.github/workflows/task_runner_docker_e2e.yml index 5c31c7fbec..023f82323f 100644 --- a/.github/workflows/task_runner_docker_e2e.yml +++ b/.github/workflows/task_runner_docker_e2e.yml @@ -66,6 +66,14 @@ jobs: pip install . pip install -r test-requirements.txt + - name: Create OpenFL image + id: create_openfl_image + run: | + echo "Creating openfl image with current repo and branch. This may take a few minutes..." + cd openfl-docker + docker build . -t openfl -f Dockerfile.base \ + --build-arg OPENFL_REVISION=https://github.com/${{ github.repository }}.git@${{ github.ref_name }} + - name: Run Task Runner E2E tests with TLS id: run_tests run: | @@ -134,6 +142,14 @@ jobs: pip install . pip install -r test-requirements.txt + - name: Create OpenFL image + id: create_openfl_image + run: | + echo "Creating openfl image with current repo and branch. This may take a few minutes..." + cd openfl-docker + docker build . -t openfl -f Dockerfile.base \ + --build-arg OPENFL_REVISION=https://github.com/${{ github.repository }}.git@${{ github.ref_name }} + - name: Run Task Runner E2E tests without TLS id: run_tests run: | @@ -202,7 +218,15 @@ jobs: pip install . pip install -r test-requirements.txt - - name: Run Task Runner E2E tests without TLS + - name: Create OpenFL image + id: create_openfl_image + run: | + echo "Creating openfl image with current repo and branch. This may take a few minutes..." + cd openfl-docker + docker build . -t openfl -f Dockerfile.base \ + --build-arg OPENFL_REVISION=https://github.com/${{ github.repository }}.git@${{ github.ref_name }} + + - name: Run Task Runner E2E tests without Client Auth id: run_tests run: | python -m pytest -s tests/end_to_end/test_suites/docker_tests.py \ diff --git a/.github/workflows/task_runner_e2e.yml b/.github/workflows/task_runner_e2e.yml index 81c98b29e2..3581517c55 100644 --- a/.github/workflows/task_runner_e2e.yml +++ b/.github/workflows/task_runner_e2e.yml @@ -272,7 +272,7 @@ jobs: pip install . pip install -r test-requirements.txt - - name: Run Task Runner E2E tests without TLS + - name: Run Task Runner E2E tests with TLS and memory logs id: run_tests run: | python -m pytest -s tests/end_to_end/test_suites/memory_logs_tests.py \ @@ -291,7 +291,7 @@ jobs: - name: Tar files id: tar_files if: ${{ always() }} - run: tar -cvf result.tar results + run: tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results - name: Upload Artifacts id: upload_artifacts diff --git a/tests/end_to_end/README.md b/tests/end_to_end/README.md index 861ecdce6b..040378804e 100644 --- a/tests/end_to_end/README.md +++ b/tests/end_to_end/README.md @@ -8,7 +8,7 @@ This project aims at integration testing of ```openfl-workspace``` using pytest tests/end_to_end ├── models # Central location for all model-related code for testing purpose ├── test_suites # Folder containing test files -├── utils # Folder containing helper files +├── utils # Folder containing fixture and helper files ├── conftest.py # Pytest framework configuration file ├── pytest.ini # Pytest initialisation file └── README.md # Readme file diff --git a/tests/end_to_end/conftest.py b/tests/end_to_end/conftest.py index 5ca588ebcc..10020c5d45 100644 --- a/tests/end_to_end/conftest.py +++ b/tests/end_to_end/conftest.py @@ -2,25 +2,17 @@ # SPDX-License-Identifier: Apache-2.0 import pytest -import collections + import os import shutil import xml.etree.ElementTree as ET import logging -import concurrent.futures + from tests.end_to_end.utils.logger import configure_logging from tests.end_to_end.utils.logger import logger as log from tests.end_to_end.utils.conftest_helper import parse_arguments -import tests.end_to_end.utils.docker_helper as dh -import tests.end_to_end.utils.federation_helper as fh -from tests.end_to_end.models import aggregator as agg_model, model_owner as mo_model -# Define a named tuple to store the objects for model owner, aggregator, and collaborators -federation_fixture = collections.namedtuple( - "federation_fixture", - "model_owner, aggregator, collaborators, workspace_path, local_bind_path", -) def pytest_addoption(parser): """ @@ -217,100 +209,3 @@ def pytest_configure(config): config.use_tls = not args.disable_tls config.log_memory_usage = args.log_memory_usage config.results_dir = config.getini("results_dir") - - -@pytest.fixture(scope="function") -def fx_federation(request): - """ - Fixture for federation. This fixture is used to create the model owner, aggregator, and collaborators. - It also creates workspace. - Assumption: OpenFL workspace is present for the model being tested. - Args: - request: pytest request object. Model name is passed as a parameter to the fixture from test cases. - Returns: - federation_fixture: Named tuple containing the objects for model owner, aggregator, and collaborators - - Note: As this is a function level fixture, thus no import is required at test level. - """ - collaborators = [] - executor = concurrent.futures.ThreadPoolExecutor() - - test_env, model_name, workspace_path, local_bind_path, agg_domain_name = fh.federation_env_setup_and_validate(request) - agg_workspace_path = os.path.join(workspace_path, "aggregator", "workspace") - - # Create model owner object and the workspace for the model - # Workspace name will be same as the model name - model_owner = mo_model.ModelOwner(model_name, request.config.log_memory_usage, workspace_path=agg_workspace_path) - - # Create workspace for given model name - fh.create_persistent_store(model_owner.name, local_bind_path) - - # Start the docker container for aggregator in case of docker environment - if test_env == "docker": - container = dh.start_docker_container( - container_name="aggregator", - workspace_path=workspace_path, - local_bind_path=local_bind_path, - ) - model_owner.container_id = container.id - - model_owner.create_workspace() - fh.add_local_workspace_permission(local_bind_path) - - # Modify the plan - plan_path = os.path.join(local_bind_path, "aggregator", "workspace", "plan") - model_owner.modify_plan( - plan_path=plan_path, - new_rounds=request.config.num_rounds, - num_collaborators=request.config.num_collaborators, - disable_client_auth=not request.config.require_client_auth, - disable_tls=not request.config.use_tls, - ) - - # Certify the workspace in case of TLS - # Register the collaborators in case of non-TLS - if request.config.use_tls: - model_owner.certify_workspace() - else: - model_owner.register_collaborators(plan_path, request.config.num_collaborators) - - # Initialize the plan - model_owner.initialize_plan(agg_domain_name=agg_domain_name) - - # Create the objects for aggregator and collaborators - # Workspace path for aggregator is uniform in case of docker or task_runner - # But, for collaborators, it is different - aggregator = agg_model.Aggregator( - agg_domain_name=agg_domain_name, - workspace_path=agg_workspace_path, - container_id=model_owner.container_id, # None in case of non-docker environment - ) - - # Generate the sign request and certify the aggregator in case of TLS - if request.config.use_tls: - aggregator.generate_sign_request() - model_owner.certify_aggregator(agg_domain_name) - - # Export the workspace - # By default the workspace will be exported to workspace.zip - model_owner.export_workspace() - - futures = [ - executor.submit( - fh.setup_collaborator, - count=i, - workspace_path=workspace_path, - local_bind_path=local_bind_path, - ) - for i in range(request.config.num_collaborators) - ] - collaborators = [f.result() for f in futures] - - # Return the federation fixture - return federation_fixture( - model_owner=model_owner, - aggregator=aggregator, - collaborators=collaborators, - workspace_path=workspace_path, - local_bind_path=local_bind_path, - ) diff --git a/tests/end_to_end/test_suites/docker_tests.py b/tests/end_to_end/test_suites/docker_tests.py index 779baaa206..6ed69b976e 100644 --- a/tests/end_to_end/test_suites/docker_tests.py +++ b/tests/end_to_end/test_suites/docker_tests.py @@ -4,6 +4,7 @@ import pytest import logging +from tests.end_to_end.utils.common_fixtures import fx_federation from tests.end_to_end.utils import federation_helper as fed_helper log = logging.getLogger(__name__) diff --git a/tests/end_to_end/test_suites/memory_logs_tests.py b/tests/end_to_end/test_suites/memory_logs_tests.py index d763c419d0..1287da3409 100644 --- a/tests/end_to_end/test_suites/memory_logs_tests.py +++ b/tests/end_to_end/test_suites/memory_logs_tests.py @@ -6,6 +6,7 @@ import os import json +from tests.end_to_end.utils.common_fixtures import fx_federation from tests.end_to_end.utils import federation_helper as fed_helper log = logging.getLogger(__name__) @@ -38,36 +39,55 @@ def test_log_memory_usage(request, fx_federation): # Setup PKI for trusted communication within the federation if request.config.use_tls: - assert fed_helper.setup_pki(fx_federation), "Failed to setup PKI for trusted communication" + assert fed_helper.setup_pki( + fx_federation + ), "Failed to setup PKI for trusted communication" # Start the federation results = fed_helper.run_federation(fx_federation) # Verify the completion of the federation run - assert fed_helper.verify_federation_run_completion(fx_federation, results, \ - num_rounds=request.config.num_rounds), "Federation completion failed" + assert fed_helper.verify_federation_run_completion( + fx_federation, results, num_rounds=request.config.num_rounds + ), "Federation completion failed" # Verify the aggregator memory logs - aggregator_memory_usage_file = os.path.join(fx_federation.workspace_path, "logs", "aggregator_memory_usage.json") - assert os.path.exists(aggregator_memory_usage_file), "Aggregator memory usage file is not available" + aggregator_memory_usage_file = os.path.join( + fx_federation.workspace_path, + "aggregator", + "workspace", + "logs", + "aggregator_memory_usage.json", + ) + assert os.path.exists( + aggregator_memory_usage_file + ), "Aggregator memory usage file is not available" # Log the aggregator memory usage details memory_usage_dict = json.load(open(aggregator_memory_usage_file)) # check memory usage entries for each round - assert len(memory_usage_dict) == request.config.num_rounds, \ - "Memory usage details are not available for all rounds" + assert ( + len(memory_usage_dict) == request.config.num_rounds + ), "Memory usage details are not available for all rounds" # check memory usage entries for each collaborator for collaborator in fx_federation.collaborators: - collaborator_memory_usage_file = os.path.join(fx_federation.workspace_path, - "logs", - f"{collaborator.collaborator_name}_memory_usage.json") + collaborator_memory_usage_file = os.path.join( + fx_federation.workspace_path, + collaborator.name, + "workspace", + "logs", + f"{collaborator.collaborator_name}_memory_usage.json", + ) - assert os.path.exists(collaborator_memory_usage_file), f"Memory usage file for collaborator {collaborator.collaborator_name} is not available" + assert os.path.exists( + collaborator_memory_usage_file + ), f"Memory usage file for collaborator {collaborator.collaborator_name} is not available" memory_usage_dict = json.load(open(collaborator_memory_usage_file)) - assert len(memory_usage_dict) == request.config.num_rounds, \ - f"Memory usage details are not available for all rounds for collaborator {collaborator.collaborator_name}" + assert ( + len(memory_usage_dict) == request.config.num_rounds + ), f"Memory usage details are not available for all rounds for collaborator {collaborator.collaborator_name}" log.info("Memory usage details are available for all participants") diff --git a/tests/end_to_end/test_suites/sample_tests.py b/tests/end_to_end/test_suites/sample_tests.py index a27bf76cbf..b5a102aa20 100644 --- a/tests/end_to_end/test_suites/sample_tests.py +++ b/tests/end_to_end/test_suites/sample_tests.py @@ -4,6 +4,7 @@ import pytest import logging +from tests.end_to_end.utils.common_fixtures import fx_federation from tests.end_to_end.utils import federation_helper as fed_helper log = logging.getLogger(__name__) diff --git a/tests/end_to_end/test_suites/task_runner_tests.py b/tests/end_to_end/test_suites/task_runner_tests.py index 077ed642dd..dd43f74275 100644 --- a/tests/end_to_end/test_suites/task_runner_tests.py +++ b/tests/end_to_end/test_suites/task_runner_tests.py @@ -4,6 +4,7 @@ import pytest import logging +from tests.end_to_end.utils.common_fixtures import fx_federation from tests.end_to_end.utils import federation_helper as fed_helper log = logging.getLogger(__name__) diff --git a/tests/end_to_end/utils/common_fixtures.py b/tests/end_to_end/utils/common_fixtures.py new file mode 100644 index 0000000000..d4912a0e0f --- /dev/null +++ b/tests/end_to_end/utils/common_fixtures.py @@ -0,0 +1,115 @@ +# Copyright 2020-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import pytest +import collections +import concurrent.futures +import os + +import tests.end_to_end.utils.docker_helper as dh +import tests.end_to_end.utils.federation_helper as fh +from tests.end_to_end.models import aggregator as agg_model, model_owner as mo_model + + +# Define a named tuple to store the objects for model owner, aggregator, and collaborators +federation_fixture = collections.namedtuple( + "federation_fixture", + "model_owner, aggregator, collaborators, workspace_path, local_bind_path", +) + + +@pytest.fixture(scope="function") +def fx_federation(request): + """ + Fixture for federation. This fixture is used to create the model owner, aggregator, and collaborators. + It also creates workspace. + Assumption: OpenFL workspace is present for the model being tested. + Args: + request: pytest request object. Model name is passed as a parameter to the fixture from test cases. + Returns: + federation_fixture: Named tuple containing the objects for model owner, aggregator, and collaborators + + Note: As this is a function level fixture, thus no import is required at test level. + """ + collaborators = [] + executor = concurrent.futures.ThreadPoolExecutor() + + test_env, model_name, workspace_path, local_bind_path, agg_domain_name = fh.federation_env_setup_and_validate(request) + agg_workspace_path = os.path.join(workspace_path, "aggregator", "workspace") + + # Create model owner object and the workspace for the model + # Workspace name will be same as the model name + model_owner = mo_model.ModelOwner(model_name, request.config.log_memory_usage, workspace_path=agg_workspace_path) + + # Create workspace for given model name + fh.create_persistent_store(model_owner.name, local_bind_path) + + # Start the docker container for aggregator in case of docker environment + if test_env == "docker": + container = dh.start_docker_container( + container_name="aggregator", + workspace_path=workspace_path, + local_bind_path=local_bind_path, + ) + model_owner.container_id = container.id + + model_owner.create_workspace() + fh.add_local_workspace_permission(local_bind_path) + + # Modify the plan + plan_path = os.path.join(local_bind_path, "aggregator", "workspace", "plan") + model_owner.modify_plan( + plan_path=plan_path, + new_rounds=request.config.num_rounds, + num_collaborators=request.config.num_collaborators, + disable_client_auth=not request.config.require_client_auth, + disable_tls=not request.config.use_tls, + ) + + # Certify the workspace in case of TLS + # Register the collaborators in case of non-TLS + if request.config.use_tls: + model_owner.certify_workspace() + else: + model_owner.register_collaborators(plan_path, request.config.num_collaborators) + + # Initialize the plan + model_owner.initialize_plan(agg_domain_name=agg_domain_name) + + # Create the objects for aggregator and collaborators + # Workspace path for aggregator is uniform in case of docker or task_runner + # But, for collaborators, it is different + aggregator = agg_model.Aggregator( + agg_domain_name=agg_domain_name, + workspace_path=agg_workspace_path, + container_id=model_owner.container_id, # None in case of non-docker environment + ) + + # Generate the sign request and certify the aggregator in case of TLS + if request.config.use_tls: + aggregator.generate_sign_request() + model_owner.certify_aggregator(agg_domain_name) + + # Export the workspace + # By default the workspace will be exported to workspace.zip + model_owner.export_workspace() + + futures = [ + executor.submit( + fh.setup_collaborator, + count=i, + workspace_path=workspace_path, + local_bind_path=local_bind_path, + ) + for i in range(request.config.num_collaborators) + ] + collaborators = [f.result() for f in futures] + + # Return the federation fixture + return federation_fixture( + model_owner=model_owner, + aggregator=aggregator, + collaborators=collaborators, + workspace_path=workspace_path, + local_bind_path=local_bind_path, + ) diff --git a/tests/end_to_end/utils/summary_helper.py b/tests/end_to_end/utils/summary_helper.py index 8fcf45cebc..d71cdd15f4 100644 --- a/tests/end_to_end/utils/summary_helper.py +++ b/tests/end_to_end/utils/summary_helper.py @@ -11,7 +11,12 @@ parser = etree.XMLParser(recover=True, encoding='utf-8') result_path = os.path.join(os.getenv("HOME"), "results") -tree = ET.parse(f"{result_path}/results.xml", parser=parser) +result_xml = os.path.join(result_path, "results.xml") +if not os.path.exists(result_xml): + print(f"Results XML file not found at {result_xml}. Exiting...") + exit(1) + +tree = ET.parse(result_xml, parser=parser) # Get the root element testsuites = tree.getroot()