diff --git a/.github/workflows/task_runner_docker_e2e.yml b/.github/workflows/task_runner_docker_e2e.yml deleted file mode 100644 index 47715e6b1e..0000000000 --- a/.github/workflows/task_runner_docker_e2e.yml +++ /dev/null @@ -1,158 +0,0 @@ ---- -#--------------------------------------------------------------------------- -# Workflow to run Task Runner E2E tests via Docker -# Authors - Noopur, Payal Chaurasiya -#--------------------------------------------------------------------------- -name: Task_Runner_E2E_via_Docker # Please do not modify the name as it is used in the composite action - -on: - workflow_dispatch: - inputs: - num_rounds: - description: "Number of rounds to train" - required: false - default: "5" - type: string - num_collaborators: - description: "Number of collaborators" - required: false - default: "2" - type: string - -permissions: - contents: read - -# Environment variables common for all the jobs -env: - NUM_ROUNDS: ${{ inputs.num_rounds || '5' }} - NUM_COLLABORATORS: ${{ inputs.num_collaborators || '2' }} - -jobs: - test_with_tls_docker: - name: tr_tls_docker - runs-on: ubuntu-22.04 - timeout-minutes: 15 - strategy: - matrix: - # There are open issues for some of the models, so excluding them for now: - # model_name: [ "torch_cnn_mnist", "keras_cnn_mnist", "torch_cnn_histology" ] - model_name: ["keras_cnn_mnist"] - python_version: ["3.9"] - fail-fast: false # do not immediately fail if one of the combinations fail - - env: - MODEL_NAME: ${{ matrix.model_name }} - PYTHON_VERSION: ${{ matrix.python_version }} - - steps: - - name: Checkout OpenFL repository - id: checkout_openfl - uses: actions/checkout@v4.1.1 - with: - fetch-depth: 2 # needed for detecting changes - submodules: "true" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Pre test run - uses: ./.github/actions/tr_pre_test_run - if: ${{ always() }} - - - name: Run Task Runner E2E tests with TLS - id: run_tests - run: | - python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ - -m task_runner_docker --model_name ${{ env.MODEL_NAME }} \ - --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} - echo "Task runner end to end test run completed" - - - name: Post test run - uses: ./.github/actions/tr_post_test_run - if: ${{ always() }} - with: - test_type: "tr_tls_docker" - - test_with_non_tls_docker: - name: tr_non_tls_docker - runs-on: ubuntu-22.04 - timeout-minutes: 15 - strategy: - matrix: - # Testing non TLS scenario only for keras_cnn_mnist model and python 3.10 - # If required, this can be extended to other models and python versions - model_name: ["keras_cnn_mnist"] - python_version: ["3.10"] - fail-fast: false # do not immediately fail if one of the combinations fail - - env: - MODEL_NAME: ${{ matrix.model_name }} - PYTHON_VERSION: ${{ matrix.python_version }} - - steps: - - name: Checkout OpenFL repository - id: checkout_openfl - uses: actions/checkout@v4.1.1 - with: - fetch-depth: 2 # needed for detecting changes - submodules: "true" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Pre test run - uses: ./.github/actions/tr_pre_test_run - if: ${{ always() }} - - - name: Run Task Runner E2E tests without TLS - id: run_tests - run: | - python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ - -m task_runner_docker --model_name ${{ env.MODEL_NAME }} \ - --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_tls - echo "Task runner end to end test run completed" - - - name: Post test run - uses: ./.github/actions/tr_post_test_run - if: ${{ always() }} - with: - test_type: "tr_non_tls_docker" - - test_with_no_client_auth_docker: - name: tr_no_client_auth_docker - runs-on: ubuntu-22.04 - timeout-minutes: 15 - strategy: - matrix: - # Testing non TLS scenario only for keras_cnn_mnist model and python 3.10 - # If required, this can be extended to other models and python versions - model_name: ["keras_cnn_mnist"] - python_version: ["3.11"] - fail-fast: false # do not immediately fail if one of the combinations fail - - env: - MODEL_NAME: ${{ matrix.model_name }} - PYTHON_VERSION: ${{ matrix.python_version }} - - steps: - - name: Checkout OpenFL repository - id: checkout_openfl - uses: actions/checkout@v4.1.1 - with: - fetch-depth: 2 # needed for detecting changes - submodules: "true" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Pre test run - uses: ./.github/actions/tr_pre_test_run - if: ${{ always() }} - - - name: Run Task Runner E2E tests without Client Auth - id: run_tests - run: | - python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ - -m task_runner_docker --model_name ${{ env.MODEL_NAME }} \ - --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_client_auth - echo "Task runner end to end test run completed" - - - name: Post test run - uses: ./.github/actions/tr_post_test_run - if: ${{ always() }} - with: - test_type: "tr_no_client_auth_docker" diff --git a/.github/workflows/task_runner_dockerized_ws_e2e.yml b/.github/workflows/task_runner_dockerized_ws_e2e.yml index d53c923ac9..e1963a584e 100644 --- a/.github/workflows/task_runner_dockerized_ws_e2e.yml +++ b/.github/workflows/task_runner_dockerized_ws_e2e.yml @@ -58,7 +58,7 @@ jobs: - name: Run Task Runner E2E tests with TLS id: run_tests run: | - python -m pytest -s tests/end_to_end/test_suites/task_runner_dockerized_ws_tests.py \ + python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ -m task_runner_dockerized_ws --model_name ${{ env.MODEL_NAME }} \ --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} echo "Task runner end to end test run completed" @@ -99,7 +99,7 @@ jobs: - name: Run Task Runner E2E tests without TLS id: run_tests run: | - python -m pytest -s tests/end_to_end/test_suites/task_runner_dockerized_ws_tests.py \ + python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ -m task_runner_dockerized_ws --model_name ${{ env.MODEL_NAME }} \ --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_tls echo "Task runner end to end test run completed" @@ -140,7 +140,7 @@ jobs: - name: Run Task Runner E2E tests without TLS id: run_tests run: | - python -m pytest -s tests/end_to_end/test_suites/task_runner_dockerized_ws_tests.py \ + python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ -m task_runner_dockerized_ws --model_name ${{ env.MODEL_NAME }} \ --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_client_auth echo "Task runner end to end test run completed" diff --git a/tests/end_to_end/README.md b/tests/end_to_end/README.md index 960396437e..191cfd0db4 100644 --- a/tests/end_to_end/README.md +++ b/tests/end_to_end/README.md @@ -55,15 +55,15 @@ For example, to run Task runner (bare metal approach) with - torch_cnn_mnist mod python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_basic --num_rounds 5 --num_collaborators 3 --model_name torch_cnn_mnist --disable_tls ``` -And, to run Task runner (via docker) with keras_cnn_mnist, 2 collaborators, 5 rounds: +And, to run Task runner (via dockerized workspace) with keras_cnn_mnist, 2 collaborators, 3 rounds: ```sh -python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_docker --num_rounds 5 --num_collaborators 2 --model_name keras_cnn_mnist +python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_dockerized_ws --num_rounds 3 --num_collaborators 2 --model_name keras_cnn_mnist ``` ### Fixture and marker mapping: -- `fx_federation_tr` - for task_runner_basic and task_runner_docker +- `fx_federation_tr` - for task_runner_basic - `fx_federation_tr_dws` - for task_runner_dockerized_ws diff --git a/tests/end_to_end/models/model_owner.py b/tests/end_to_end/models/model_owner.py index 31793fbf03..8e66968ea9 100644 --- a/tests/end_to_end/models/model_owner.py +++ b/tests/end_to_end/models/model_owner.py @@ -52,8 +52,7 @@ def create_workspace(self): log.info(f"Creating workspace for model {self.model_name} at the path: {self.workspace_path}") error_msg = "Failed to create the workspace" - # Docker environment requires the path to be relative - ws_path = self.workspace_path.lstrip('/') if os.getenv("TEST_ENV") == "task_runner_docker" else self.workspace_path + ws_path = self.workspace_path return_code, output, error = fh.run_command( f"fx workspace create --prefix {ws_path} --template {self.model_name}", diff --git a/tests/end_to_end/pytest.ini b/tests/end_to_end/pytest.ini index 39567147e7..ed865c99c6 100644 --- a/tests/end_to_end/pytest.ini +++ b/tests/end_to_end/pytest.ini @@ -7,7 +7,6 @@ log_level = INFO markers = log_memory_usage: mark a test as a log memory usage test. task_runner_basic: mark a test as a task runner basic test. - task_runner_docker: mark a test as a task runner docker test. - task_runner_dockerized_ws: mark a test as a task runner dockerized ws test. + task_runner_dockerized_ws: mark a test as a task runner dockerized workspace test. asyncio_mode=auto asyncio_default_fixture_loop_scope="function" diff --git a/tests/end_to_end/test_suites/sample_tests.py b/tests/end_to_end/test_suites/sample_tests.py index 58086a519e..981b214e49 100644 --- a/tests/end_to_end/test_suites/sample_tests.py +++ b/tests/end_to_end/test_suites/sample_tests.py @@ -40,22 +40,6 @@ def test_federation_basic(request, fx_federation_tr): ), "Federation completion failed" -@pytest.mark.task_runner_docker -def test_federation_via_docker(request, fx_federation_tr): - """ - Add a proper docstring here. - """ - log.info(f"Running sample model test {fx_federation_tr}") - - # Start the federation - results = fed_helper.run_federation(fx_federation_tr) - - # Verify the completion of the federation run - assert fed_helper.verify_federation_run_completion( - fx_federation_tr, results, num_rounds=request.config.num_rounds - ), "Federation completion failed" - - @pytest.mark.task_runner_dockerized_ws def test_federation_via_dockerized_workspace(request, fx_federation_tr_dws): """ diff --git a/tests/end_to_end/test_suites/task_runner_dockerized_ws_tests.py b/tests/end_to_end/test_suites/task_runner_dockerized_ws_tests.py deleted file mode 100644 index 4adbc31dbd..0000000000 --- a/tests/end_to_end/test_suites/task_runner_dockerized_ws_tests.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2020-2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -import pytest -import logging - -from tests.end_to_end.utils.common_fixtures import fx_federation_tr_dws -from tests.end_to_end.utils import federation_helper as fed_helper - -log = logging.getLogger(__name__) - - -# NOTE: This test file contains the test cases for the task runner federation using dockerized workspace approach. - -@pytest.mark.task_runner_dockerized_ws -def test_federation_via_dockerized_workspace(request, fx_federation_tr_dws): - """ - Test federation via dockerized workspace. - Args: - request (Fixture): Pytest fixture - fx_federation (Fixture): Pytest fixture - """ - # Start the federation - results = fed_helper.run_federation_for_dws(fx_federation_tr_dws, use_tls=request.config.use_tls) - - log.info(f"Federation run completed successfully with {results}") - # Verify the completion of the federation run - assert fed_helper.verify_federation_run_completion(fx_federation_tr_dws, results, request.config.num_rounds), "Federation completion failed" diff --git a/tests/end_to_end/test_suites/task_runner_tests.py b/tests/end_to_end/test_suites/task_runner_tests.py index 6363b7e452..fb4200bdec 100644 --- a/tests/end_to_end/test_suites/task_runner_tests.py +++ b/tests/end_to_end/test_suites/task_runner_tests.py @@ -4,18 +4,19 @@ import pytest import logging -from tests.end_to_end.utils.common_fixtures import fx_federation_tr +from tests.end_to_end.utils.common_fixtures import fx_federation_tr, fx_federation_tr_dws from tests.end_to_end.utils import federation_helper as fed_helper log = logging.getLogger(__name__) -# NOTE: This test file contains the test cases for the task runner federation using bare metal and docker approaches. - @pytest.mark.task_runner_basic def test_federation_via_native(request, fx_federation_tr): """ Test federation via native task runner. + Args: + request (Fixture): Pytest fixture + fx_federation_tr (Fixture): Pytest fixture for native task runner """ # Start the federation results = fed_helper.run_federation(fx_federation_tr) @@ -26,18 +27,16 @@ def test_federation_via_native(request, fx_federation_tr): ), "Federation completion failed" -@pytest.mark.task_runner_docker -def test_federation_via_docker(request, fx_federation_tr): +@pytest.mark.task_runner_dockerized_ws +def test_federation_via_dockerized_workspace(request, fx_federation_tr_dws): """ - Test federation via docker. + Test federation via dockerized workspace. Args: request (Fixture): Pytest fixture - fx_federation_tr (Fixture): Pytest fixture + fx_federation_tr_dws (Fixture): Pytest fixture for dockerized workspace """ # Start the federation - results = fed_helper.run_federation(fx_federation_tr) + results = fed_helper.run_federation_for_dws(fx_federation_tr_dws, use_tls=request.config.use_tls) # Verify the completion of the federation run - assert fed_helper.verify_federation_run_completion( - fx_federation_tr, results, request.config.num_rounds - ), "Federation completion failed" + assert fed_helper.verify_federation_run_completion(fx_federation_tr_dws, results, request.config.num_rounds), "Federation completion failed" diff --git a/tests/end_to_end/utils/common_fixtures.py b/tests/end_to_end/utils/common_fixtures.py index d7da06c9de..c00eb4b969 100644 --- a/tests/end_to_end/utils/common_fixtures.py +++ b/tests/end_to_end/utils/common_fixtures.py @@ -51,9 +51,9 @@ def fx_federation_tr(request): """ test_env = fh.get_test_env_from_markers(request) - if test_env not in ["task_runner_docker", "task_runner_basic"]: + if test_env != "task_runner_basic": raise ValueError( - "Fixture fx_federation_tr is only supported for task_runner_basic and task_runner_docker markers" + "Fixture fx_federation_tr is only supported for task_runner_basic marker" ) collaborators = [] @@ -74,15 +74,6 @@ def fx_federation_tr(request): # Create workspace for given model name fh.create_persistent_store(model_owner.name, local_bind_path) - # Start the docker container for aggregator in case of docker environment - if test_env == "task_runner_docker": - container = dh.start_docker_container( - container_name="aggregator", - workspace_path=workspace_path, - local_bind_path=local_bind_path, - ) - model_owner.container_id = container.id - model_owner.create_workspace() fh.add_local_workspace_permission(local_bind_path) diff --git a/tests/end_to_end/utils/federation_helper.py b/tests/end_to_end/utils/federation_helper.py index 2247ddcd47..8a3c87b940 100644 --- a/tests/end_to_end/utils/federation_helper.py +++ b/tests/end_to_end/utils/federation_helper.py @@ -384,11 +384,7 @@ def _verify_completion_for_participant( test_env = get_test_env_from_markers() # In case of docker environment, get the logs from local path which is mounted to the container - if test_env == "task_runner_docker": - result_file = constants.AGG_COL_RESULT_FILE.format( - local_bind_path, participant.name - ) - elif test_env == "task_runner_dockerized_ws": + if test_env == "task_runner_dockerized_ws": result_file = constants.AGG_COL_RESULT_FILE.format( local_bind_path, participant.name ) @@ -448,16 +444,14 @@ def get_test_env_from_markers(request=None): # Determine the test type based on the markers markers = [m.name for m in request.node.iter_markers()] - if "task_runner_docker" in markers: - test_env = "task_runner_docker" - elif "task_runner_basic" in markers: + if "task_runner_basic" in markers: test_env = "task_runner_basic" elif "task_runner_dockerized_ws" in markers: test_env = "task_runner_dockerized_ws" else: raise ValueError( "Invalid test environment. Provide one of the valid markers: " - "task_runner_docker, task_runner_basic, task_runner_dockerized_ws" + "task_runner_basic and task_runner_dockerized_ws" ) os.environ["TEST_ENV"] = test_env @@ -488,7 +482,7 @@ def federation_env_setup_and_validate(request): ) workspace_path = local_bind_path - if test_env in ["task_runner_docker", "task_runner_dockerized_ws"]: + if test_env == "task_runner_dockerized_ws": agg_domain_name = "aggregator" # Cleanup docker containers @@ -496,16 +490,6 @@ def federation_env_setup_and_validate(request): dh.remove_docker_network() dh.create_docker_network() - # Note: In case of dockerized workspace, image name would be same as workspace name and to be created at later stage. - if test_env == "task_runner_docker": - # Check if the docker image and network exists - dh.check_docker_image() - - # Absolute path is required for docker - workspace_path = os.path.join( - "/", request.config.results_dir, request.config.model_name - ) - log.info( f"Running federation setup using {test_env} API on single machine with below configurations:\n" f"\tNumber of collaborators: {request.config.num_collaborators}\n" @@ -599,13 +583,7 @@ def run_command( return_code, output, error = 0, None, None error_msg = error_msg or "Failed to run the command" - is_docker = ( - True - if (os.getenv("TEST_ENV") == "task_runner_docker" or run_for_dockerized_ws) - else False - ) - - if is_docker and container_id: + if run_for_dockerized_ws and container_id: log.debug("Running command in docker container") if len(workspace_path): docker_command = f"docker exec -w {workspace_path} {container_id} sh -c " @@ -628,7 +606,7 @@ def run_command( log.info(f"Running command: {command}") log.debug("Running command on local machine") - if run_in_background and not is_docker: + if run_in_background and not run_for_dockerized_ws: bg_file = open(bg_file, "w", buffering=1) ssh.run_command_background( command, @@ -693,19 +671,6 @@ def setup_collaborator(count, workspace_path, local_bind_path): f"Failed to create persistent store for {collaborator.name}: {e}" ) - try: - if test_env == "task_runner_docker": - container = dh.start_docker_container( - container_name=collaborator.name, - workspace_path=workspace_path, - local_bind_path=local_bind_path, - ) - collaborator.container_id = container.id - except Exception as e: - raise ex.DockerException( - f"Failed to start {collaborator.name} docker environment: {e}" - ) - try: local_col_ws_path = constants.COL_WORKSPACE_PATH.format( local_bind_path, collaborator.name