diff --git a/.github/actions/task_runner_e2e/post_run.yml b/.github/actions/task_runner_e2e/post_run.yml new file mode 100644 index 0000000000..797f28fa79 --- /dev/null +++ b/.github/actions/task_runner_e2e/post_run.yml @@ -0,0 +1,43 @@ +--- +#--------------------------------------------------------------------------- +# Composite Action to run post-test functions for task runner end to end tests +# Includes: +# - Print test summary +# - Create tar file +# - Upload artifacts +# +# Authors - Noopur, Payal Chaurasiya +#--------------------------------------------------------------------------- +name: 'Post-Test Functions' +description: 'Run post-test functions' +inputs: + test_type: + description: 'Test type' + required: true + +runs: + using: 'composite' + steps: + - name: Print test summary + id: print_test_summary + if: ${{ always() }} + run: | + export PYTHONPATH="$PYTHONPATH:." + python tests/end_to_end/utils/summary_helper.py + echo "Test summary printed" + shell: bash + + - name: Create Tar (exclude cert and data folders) + id: tar_files + if: ${{ always() }} + run: | + tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results + shell: bash + + - name: Upload Artifacts + id: upload_artifacts + uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: ${{ inputs.test_type }}_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} + path: result.tar diff --git a/.github/actions/task_runner_e2e/pre_run.yml b/.github/actions/task_runner_e2e/pre_run.yml new file mode 100644 index 0000000000..2c366bfab1 --- /dev/null +++ b/.github/actions/task_runner_e2e/pre_run.yml @@ -0,0 +1,39 @@ +--- +#--------------------------------------------------------------------------- +# Composite Action to run pre-test functions for task runner end to end tests +# Includes: +# - Set up Python +# - Install dependencies +# +# Authors - Noopur, Payal Chaurasiya +#--------------------------------------------------------------------------- +name: 'Pre-Test Functions' +description: 'Run pre-test functions' + +runs: + using: 'composite' + steps: + - name: Set up Python + id: setup_python + uses: actions/setup-python@v3 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install dependencies + id: install_dependencies + run: | + python -m pip install --upgrade pip + pip install . + pip install -r test-requirements.txt + shell: bash + + - name: Create OpenFL image + if: ${{ github.workflow == 'Task_Runner_E2E_via_Docker' }} + id: create_openfl_image + run: | + echo "Creating openfl image with current repo and branch. This may take a few minutes..." + cd openfl-docker + docker build . -t openfl -f Dockerfile.base \ + --build-arg OPENFL_REVISION=https://github.com/${{ github.repository }}.git@${{ github.ref_name }} + echo "OpenFL image created successfully" + shell: bash diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 52b9a477a0..e5f1427bec 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -3,7 +3,7 @@ # Workflow to run Task Runner end to end tests using bare metal approach # Authors - Noopur, Payal Chaurasiya #--------------------------------------------------------------------------- -name: Task Runner E2E +name: Task_Runner_E2E # Please do not modify the name as it is used in the composite action on: schedule: @@ -55,18 +55,9 @@ jobs: submodules: "true" token: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Python - id: setup_python - uses: actions/setup-python@v3 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - id: install_dependencies - run: | - python -m pip install --upgrade pip - pip install . - pip install -r test-requirements.txt + - name: Pre test run + uses: ./.github/actions/task_runner_e2e/pre_test_run + if: ${{ always() }} - name: Run Task Runner E2E tests with TLS id: run_tests @@ -76,27 +67,11 @@ jobs: --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} echo "Task runner end to end test run completed" - - name: Print test summary - id: print_test_summary - if: ${{ always() }} - run: | - export PYTHONPATH="$PYTHONPATH:." - python tests/end_to_end/utils/summary_helper.py - echo "Test summary printed" - - - name: Create Tar (exclude cert and data folders) - id: tar_files - if: ${{ always() }} - run: | - tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results - - - name: Upload Artifacts - id: upload_artifacts - uses: actions/upload-artifact@v4 + - name: Post test run + uses: ./.github/actions/task_runner_e2e/post_test_run if: ${{ always() }} with: - name: tr_tls_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} - path: result.tar + run_type: "tr_tls" test_with_non_tls: name: tr_non_tls @@ -123,18 +98,9 @@ jobs: submodules: "true" token: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Python - id: setup_python - uses: actions/setup-python@v3 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - id: install_dependencies - run: | - python -m pip install --upgrade pip - pip install . - pip install -r test-requirements.txt + - name: Pre test run + uses: ./.github/actions/task_runner_e2e/pre_test_run + if: ${{ always() }} - name: Run Task Runner E2E tests without TLS id: run_tests @@ -144,27 +110,11 @@ jobs: --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_tls echo "Task runner end to end test run completed" - - name: Print test summary - id: print_test_summary - if: ${{ always() }} - run: | - export PYTHONPATH="$PYTHONPATH:." - python tests/end_to_end/utils/summary_helper.py - echo "Test summary printed" - - - name: Create Tar (exclude cert and data folders) - id: tar_files - if: ${{ always() }} - run: | - tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results - - - name: Upload Artifacts - id: upload_artifacts - uses: actions/upload-artifact@v4 + - name: Post test run + uses: ./.github/actions/task_runner_e2e/post_test_run if: ${{ always() }} with: - name: tr_non_tls_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} - path: result.tar + run_type: "tr_non_tls" test_with_no_client_auth: name: tr_no_client_auth @@ -191,18 +141,9 @@ jobs: submodules: "true" token: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Python - id: setup_python - uses: actions/setup-python@v3 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - id: install_dependencies - run: | - python -m pip install --upgrade pip - pip install . - pip install -r test-requirements.txt + - name: Pre test run + uses: ./.github/actions/task_runner_e2e/pre_test_run + if: ${{ always() }} - name: Run Task Runner E2E tests without TLS id: run_tests @@ -212,27 +153,11 @@ jobs: --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_client_auth echo "Task runner end to end test run completed" - - name: Print test summary - id: print_test_summary - if: ${{ always() }} - run: | - export PYTHONPATH="$PYTHONPATH:." - python tests/end_to_end/utils/summary_helper.py - echo "Test summary printed" - - - name: Create Tar (exclude cert and data folders) - id: tar_files - if: ${{ always() }} - run: | - tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results - - - name: Upload Artifacts - id: upload_artifacts - uses: actions/upload-artifact@v4 + - name: Post test run + uses: ./.github/actions/task_runner_e2e/post_test_run if: ${{ always() }} with: - name: tr_no_client_auth_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} - path: result.tar + run_type: "tr_no_client_auth" test_memory_logs: name: tr_tls_memory_logs @@ -259,18 +184,9 @@ jobs: submodules: "true" token: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Python - id: setup_python - uses: actions/setup-python@v3 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - id: install_dependencies - run: | - python -m pip install --upgrade pip - pip install . - pip install -r test-requirements.txt + - name: Pre test run + uses: ./.github/actions/task_runner_e2e/pre_test_run + if: ${{ always() }} - name: Run Task Runner E2E tests with TLS and memory logs id: run_tests @@ -280,23 +196,8 @@ jobs: --num_collaborators ${{ env.NUM_COLLABORATORS }} --log_memory_usage echo "Task runner memory logs test run completed" - - name: Print test summary - id: print_test_summary - if: ${{ always() }} - run: | - export PYTHONPATH="$PYTHONPATH:." - python tests/end_to_end/utils/summary_helper.py - echo "Test summary printed" - - - name: Tar files - id: tar_files - if: ${{ always() }} - run: tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results - - - name: Upload Artifacts - id: upload_artifacts - uses: actions/upload-artifact@v4 + - name: Post test run + uses: ./.github/actions/task_runner_e2e/post_test_run if: ${{ always() }} with: - name: tr_tls_memory_logs_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} - path: result.tar + run_type: "tr_tls_memory_logs" diff --git a/.github/workflows/task_runner_docker_e2e.yml b/.github/workflows/task_runner_docker_e2e.yml index 69d1e7b895..41e84684f5 100644 --- a/.github/workflows/task_runner_docker_e2e.yml +++ b/.github/workflows/task_runner_docker_e2e.yml @@ -3,7 +3,7 @@ # Workflow to run Task Runner E2E tests via Docker # Authors - Noopur, Payal Chaurasiya #--------------------------------------------------------------------------- -name: Task Runner E2E via Docker +name: Task_Runner_E2E_via_Docker # Please do not modify the name as it is used in the composite action on: workflow_dispatch: @@ -53,26 +53,9 @@ jobs: submodules: "true" token: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Python - id: setup_python - uses: actions/setup-python@v3 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - id: install_dependencies - run: | - python -m pip install --upgrade pip - pip install . - pip install -r test-requirements.txt - - - name: Create OpenFL image - id: create_openfl_image - run: | - echo "Creating openfl image with current repo and branch. This may take a few minutes..." - cd openfl-docker - docker build . -t openfl -f Dockerfile.base \ - --build-arg OPENFL_REVISION=https://github.com/${{ github.repository }}.git@${{ github.ref_name }} + - name: Pre test run + uses: ./.github/actions/task_runner_e2e/pre_test_run + if: ${{ always() }} - name: Run Task Runner E2E tests with TLS id: run_tests @@ -82,27 +65,11 @@ jobs: --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} echo "Task runner end to end test run completed" - - name: Print test summary - id: print_test_summary - if: ${{ always() }} - run: | - export PYTHONPATH="$PYTHONPATH:." - python tests/end_to_end/utils/summary_helper.py - echo "Test summary printed" - - - name: Create Tar (exclude cert and data folders) - id: tar_files - if: ${{ always() }} - run: | - tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results - - - name: Upload Artifacts - id: upload_artifacts - uses: actions/upload-artifact@v4 + - name: Post test run + uses: ./.github/actions/task_runner_e2e/post_test_run if: ${{ always() }} with: - name: tr_tls_docker_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} - path: result.tar + run_type: "tr_tls_docker" test_with_non_tls_docker: name: tr_non_tls_docker @@ -129,26 +96,9 @@ jobs: submodules: "true" token: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Python - id: setup_python - uses: actions/setup-python@v3 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - id: install_dependencies - run: | - python -m pip install --upgrade pip - pip install . - pip install -r test-requirements.txt - - - name: Create OpenFL image - id: create_openfl_image - run: | - echo "Creating openfl image with current repo and branch. This may take a few minutes..." - cd openfl-docker - docker build . -t openfl -f Dockerfile.base \ - --build-arg OPENFL_REVISION=https://github.com/${{ github.repository }}.git@${{ github.ref_name }} + - name: Pre test run + uses: ./.github/actions/task_runner_e2e/pre_test_run + if: ${{ always() }} - name: Run Task Runner E2E tests without TLS id: run_tests @@ -158,27 +108,11 @@ jobs: --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_tls echo "Task runner end to end test run completed" - - name: Print test summary - id: print_test_summary - if: ${{ always() }} - run: | - export PYTHONPATH="$PYTHONPATH:." - python tests/end_to_end/utils/summary_helper.py - echo "Test summary printed" - - - name: Create Tar (exclude cert and data folders) - id: tar_files - if: ${{ always() }} - run: | - tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results - - - name: Upload Artifacts - id: upload_artifacts - uses: actions/upload-artifact@v4 + - name: Post test run + uses: ./.github/actions/task_runner_e2e/post_test_run if: ${{ always() }} with: - name: tr_non_tls_docker_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} - path: result.tar + run_type: "tr_non_tls_docker" test_with_no_client_auth_docker: name: tr_no_client_auth_docker @@ -205,26 +139,9 @@ jobs: submodules: "true" token: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Python - id: setup_python - uses: actions/setup-python@v3 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - id: install_dependencies - run: | - python -m pip install --upgrade pip - pip install . - pip install -r test-requirements.txt - - - name: Create OpenFL image - id: create_openfl_image - run: | - echo "Creating openfl image with current repo and branch. This may take a few minutes..." - cd openfl-docker - docker build . -t openfl -f Dockerfile.base \ - --build-arg OPENFL_REVISION=https://github.com/${{ github.repository }}.git@${{ github.ref_name }} + - name: Pre test run + uses: ./.github/actions/task_runner_e2e/pre_test_run + if: ${{ always() }} - name: Run Task Runner E2E tests without Client Auth id: run_tests @@ -234,24 +151,8 @@ jobs: --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_client_auth echo "Task runner end to end test run completed" - - name: Print test summary - id: print_test_summary - if: ${{ always() }} - run: | - export PYTHONPATH="$PYTHONPATH:." - python tests/end_to_end/utils/summary_helper.py - echo "Test summary printed" - - - name: Create Tar (exclude cert and data folders) - id: tar_files - if: ${{ always() }} - run: | - tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results - - - name: Upload Artifacts - id: upload_artifacts - uses: actions/upload-artifact@v4 + - name: Post test run + uses: ./.github/actions/task_runner_e2e/post_test_run if: ${{ always() }} with: - name: tr_no_client_auth_docker_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} - path: result.tar + run_type: "tr_no_client_auth_docker" diff --git a/.github/workflows/task_runner_dockerized_ws_e2e.yml b/.github/workflows/task_runner_dockerized_ws_e2e.yml index 8aec128032..1004ac9b81 100644 --- a/.github/workflows/task_runner_dockerized_ws_e2e.yml +++ b/.github/workflows/task_runner_dockerized_ws_e2e.yml @@ -3,7 +3,7 @@ # Workflow to run Task Runner end to end tests using dockerized workspace # Authors - Noopur, Payal Chaurasiya #--------------------------------------------------------------------------- -name: Task Runner E2E via Dockerized Workspace +name: Task_Runner_E2E_via_Dockerized_Workspace # Please do not modify the name as it is used in the composite action on: workflow_dispatch: @@ -53,18 +53,9 @@ jobs: submodules: "true" token: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Python - id: setup_python - uses: actions/setup-python@v3 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - id: install_dependencies - run: | - python -m pip install --upgrade pip - pip install . - pip install -r test-requirements.txt + - name: Pre test run + uses: ./.github/actions/task_runner_e2e/pre_test_run + if: ${{ always() }} - name: Run Task Runner E2E tests with TLS id: run_tests @@ -74,27 +65,11 @@ jobs: --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} echo "Task runner end to end test run completed" - - name: Print test summary - id: print_test_summary - if: ${{ always() }} - run: | - export PYTHONPATH="$PYTHONPATH:." - python tests/end_to_end/utils/summary_helper.py - echo "Test summary printed" - - - name: Create Tar (exclude cert and data folders) - id: tar_files - if: ${{ always() }} - run: | - tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results - - - name: Upload Artifacts - id: upload_artifacts - uses: actions/upload-artifact@v4 + - name: Post test run + uses: ./.github/actions/task_runner_e2e/post_test_run if: ${{ always() }} with: - name: tr_tls_dockerized_ws_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} - path: result.tar + run_type: "tr_tls_dockerized_ws" test_with_non_tls_dockerized_ws: name: tr_non_tls_dockerized_ws @@ -121,18 +96,9 @@ jobs: submodules: "true" token: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Python - id: setup_python - uses: actions/setup-python@v3 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - id: install_dependencies - run: | - python -m pip install --upgrade pip - pip install . - pip install -r test-requirements.txt + - name: Pre test run + uses: ./.github/actions/task_runner_e2e/pre_test_run + if: ${{ always() }} - name: Run Task Runner E2E tests without TLS id: run_tests @@ -142,27 +108,11 @@ jobs: --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_tls echo "Task runner end to end test run completed" - - name: Print test summary - id: print_test_summary - if: ${{ always() }} - run: | - export PYTHONPATH="$PYTHONPATH:." - python tests/end_to_end/utils/summary_helper.py - echo "Test summary printed" - - - name: Create Tar (exclude cert and data folders) - id: tar_files - if: ${{ always() }} - run: | - tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results - - - name: Upload Artifacts - id: upload_artifacts - uses: actions/upload-artifact@v4 + - name: Post test run + uses: ./.github/actions/task_runner_e2e/post_test_run if: ${{ always() }} with: - name: tr_non_tls_dockerized_ws_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} - path: result.tar + run_type: "tr_non_tls_dockerized_ws" test_with_no_client_auth_dockerized_ws: name: tr_no_client_auth_dockerized_ws @@ -189,18 +139,9 @@ jobs: submodules: "true" token: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Python - id: setup_python - uses: actions/setup-python@v3 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - id: install_dependencies - run: | - python -m pip install --upgrade pip - pip install . - pip install -r test-requirements.txt + - name: Pre test run + uses: ./.github/actions/task_runner_e2e/pre_test_run + if: ${{ always() }} - name: Run Task Runner E2E tests without TLS id: run_tests @@ -210,24 +151,8 @@ jobs: --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_client_auth echo "Task runner end to end test run completed" - - name: Print test summary - id: print_test_summary - if: ${{ always() }} - run: | - export PYTHONPATH="$PYTHONPATH:." - python tests/end_to_end/utils/summary_helper.py - echo "Test summary printed" - - - name: Create Tar (exclude cert and data folders) - id: tar_files - if: ${{ always() }} - run: | - tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results - - - name: Upload Artifacts - id: upload_artifacts - uses: actions/upload-artifact@v4 + - name: Post test run + uses: ./.github/actions/task_runner_e2e/post_test_run if: ${{ always() }} with: - name: tr_no_client_auth_dockerized_ws_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} - path: result.tar + run_type: "tr_no_client_auth_dockerized_ws" diff --git a/tests/end_to_end/README.md b/tests/end_to_end/README.md index 040378804e..960396437e 100644 --- a/tests/end_to_end/README.md +++ b/tests/end_to_end/README.md @@ -49,12 +49,24 @@ Below parameters are available for modification: 4. --disable_tls - to disable TLS communication (by default it is enabled) 5. --disable_client_auth - to disable the client authentication (by default it is enabled) -For example, to run Task runner with - torch_cnn_mnist model, 3 collaborators, 5 rounds and non-TLS scenario: +For example, to run Task runner (bare metal approach) with - torch_cnn_mnist model, 3 collaborators, 5 rounds and non-TLS scenario: ```sh -python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py --num_rounds 5 --num_collaborators 3 --model_name torch_cnn_mnist --disable_tls +python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_basic --num_rounds 5 --num_collaborators 3 --model_name torch_cnn_mnist --disable_tls ``` +And, to run Task runner (via docker) with keras_cnn_mnist, 2 collaborators, 5 rounds: + +```sh +python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_docker --num_rounds 5 --num_collaborators 2 --model_name keras_cnn_mnist +``` + +### Fixture and marker mapping: + +- `fx_federation_tr` - for task_runner_basic and task_runner_docker +- `fx_federation_tr_dws` - for task_runner_dockerized_ws + + ### Output Structure ``` diff --git a/tests/end_to_end/test_suites/sample_tests.py b/tests/end_to_end/test_suites/sample_tests.py index 39179b44c6..58086a519e 100644 --- a/tests/end_to_end/test_suites/sample_tests.py +++ b/tests/end_to_end/test_suites/sample_tests.py @@ -4,7 +4,10 @@ import pytest import logging -from tests.end_to_end.utils.common_fixtures import fx_federation +from tests.end_to_end.utils.common_fixtures import ( + fx_federation_tr, + fx_federation_tr_dws, +) from tests.end_to_end.utils import federation_helper as fed_helper log = logging.getLogger(__name__) @@ -13,22 +16,59 @@ # Task Runner API Test function for federation run using sample_model # 1. Create OpenFL workspace, if not present for the model and add relevant dataset and its path in plan/data.yaml # 2. Append the model name to ModelName enum in tests/end_to_end/utils/constants.py -# 3. Add the model name to tests/end_to_end/pytest.ini marker, if not present -# 4. Use fx_federation fixture in the test function - it will provide the federation object. -# 5. Fixture will contain - model_owner, aggregator, collaborators, model_name, workspace_path, results_dir -# 6. Setup PKI for trusted communication within the federation -# 7. Start the federation using aggregator and given no of collaborators. -# 8. Verify the completion of the federation run. - -@pytest.mark.sample_model_name -def test_sample_model_name(fx_federation): +# 3. a. Use fx_federation_tr fixture for task runner with bare metal or docker approach. +# 3. b. Use fx_federation_tr_dws fixture for task runner with dockerized workspace approach. +# 4. Fixture will contain - model_owner, aggregator, collaborators, workspace_path, local_bind_path +# 5. Setup PKI for trusted communication within the federation based on TLS flag. +# 6. Start the federation using aggregator and given no of collaborators. +# 7. Verify the completion of the federation run. + + +@pytest.mark.task_runner_basic +def test_federation_basic(request, fx_federation_tr): + """ + Add a proper docstring here. + """ + log.info(f"Running sample model test {fx_federation_tr}") + + # Start the federation + results = fed_helper.run_federation(fx_federation_tr) + + # Verify the completion of the federation run + assert fed_helper.verify_federation_run_completion( + fx_federation_tr, results, num_rounds=request.config.num_rounds + ), "Federation completion failed" + + +@pytest.mark.task_runner_docker +def test_federation_via_docker(request, fx_federation_tr): + """ + Add a proper docstring here. + """ + log.info(f"Running sample model test {fx_federation_tr}") + + # Start the federation + results = fed_helper.run_federation(fx_federation_tr) + + # Verify the completion of the federation run + assert fed_helper.verify_federation_run_completion( + fx_federation_tr, results, num_rounds=request.config.num_rounds + ), "Federation completion failed" + + +@pytest.mark.task_runner_dockerized_ws +def test_federation_via_dockerized_workspace(request, fx_federation_tr_dws): """ Add a proper docstring here. """ - log.info(f"Running sample model test {fx_federation.model_name}") + log.info(f"Running sample model test {fx_federation_tr_dws}") # Start the federation - results = fed_helper.run_federation(fx_federation) + results = fed_helper.run_federation( + fx_federation_tr_dws, use_tls=request.config.use_tls + ) # Verify the completion of the federation run - assert fed_helper.verify_federation_run_completion(fx_federation, results), "Federation completion failed" + assert fed_helper.verify_federation_run_completion( + fx_federation_tr_dws, results, request.config.num_rounds + ), "Federation completion failed" diff --git a/tests/end_to_end/test_suites/task_runner_dockerized_ws_tests.py b/tests/end_to_end/test_suites/task_runner_dockerized_ws_tests.py index 3620f89612..cf9976dd86 100644 --- a/tests/end_to_end/test_suites/task_runner_dockerized_ws_tests.py +++ b/tests/end_to_end/test_suites/task_runner_dockerized_ws_tests.py @@ -3,9 +3,7 @@ import pytest import logging -import concurrent.futures -import tests.end_to_end.utils.ssh_helper as ssh from tests.end_to_end.utils.common_fixtures import fx_federation_tr_dws from tests.end_to_end.utils import federation_helper as fed_helper diff --git a/tests/end_to_end/utils/federation_helper.py b/tests/end_to_end/utils/federation_helper.py index 290f647687..4c36fc04c9 100644 --- a/tests/end_to_end/utils/federation_helper.py +++ b/tests/end_to_end/utils/federation_helper.py @@ -492,6 +492,8 @@ def federation_env_setup_and_validate(request): workspace_path = local_bind_path if test_env in ["task_runner_docker", "task_runner_dockerized_ws"]: + + agg_domain_name = "aggregator" # Cleanup docker containers dh.cleanup_docker_containers() dh.remove_docker_network() @@ -501,7 +503,6 @@ def federation_env_setup_and_validate(request): if test_env == "task_runner_docker": # Check if the docker image and network exists dh.check_docker_image() - agg_domain_name = "aggregator" # Absolute path is required for docker workspace_path = os.path.join(