diff --git a/.github/workflows/task_runner_e2e.yml b/.github/workflows/task_runner_e2e.yml index 81c98b29e2..77b461b70c 100644 --- a/.github/workflows/task_runner_e2e.yml +++ b/.github/workflows/task_runner_e2e.yml @@ -30,209 +30,209 @@ env: NUM_COLLABORATORS: ${{ inputs.num_collaborators || '2' }} jobs: - test_with_tls: - name: tr_tls - runs-on: ubuntu-22.04 - timeout-minutes: 15 - strategy: - matrix: - # There are open issues for some of the models, so excluding them for now: - # model_name: [ "torch_cnn_mnist", "keras_cnn_mnist", "torch_cnn_histology" ] - model_name: ["torch_cnn_mnist", "keras_cnn_mnist"] - python_version: ["3.9", "3.10", "3.11"] - fail-fast: false # do not immediately fail if one of the combinations fail - - env: - MODEL_NAME: ${{ matrix.model_name }} - PYTHON_VERSION: ${{ matrix.python_version }} - - steps: - - name: Checkout OpenFL repository - id: checkout_openfl - uses: actions/checkout@v4.1.1 - with: - fetch-depth: 2 # needed for detecting changes - submodules: "true" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Python - id: setup_python - uses: actions/setup-python@v3 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - id: install_dependencies - run: | - python -m pip install --upgrade pip - pip install . - pip install -r test-requirements.txt - - - name: Run Task Runner E2E tests with TLS - id: run_tests - run: | - python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ - -m ${{ env.MODEL_NAME }} --model_name ${{ env.MODEL_NAME }} \ - --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} - echo "Task runner end to end test run completed" - - - name: Print test summary - id: print_test_summary - if: ${{ always() }} - run: | - export PYTHONPATH="$PYTHONPATH:." - python tests/end_to_end/utils/summary_helper.py - echo "Test summary printed" - - - name: Create Tar (exclude cert and data folders) - id: tar_files - if: ${{ always() }} - run: | - tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results - - - name: Upload Artifacts - id: upload_artifacts - uses: actions/upload-artifact@v4 - if: ${{ always() }} - with: - name: tr_tls_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} - path: result.tar - - test_with_non_tls: - name: tr_non_tls - runs-on: ubuntu-22.04 - timeout-minutes: 15 - strategy: - matrix: - # Testing non TLS scenario only for torch_cnn_mnist model and python 3.10 - # If required, this can be extended to other models and python versions - model_name: ["torch_cnn_mnist"] - python_version: ["3.10"] - fail-fast: false # do not immediately fail if one of the combinations fail - - env: - MODEL_NAME: ${{ matrix.model_name }} - PYTHON_VERSION: ${{ matrix.python_version }} - - steps: - - name: Checkout OpenFL repository - id: checkout_openfl - uses: actions/checkout@v4.1.1 - with: - fetch-depth: 2 # needed for detecting changes - submodules: "true" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Python - id: setup_python - uses: actions/setup-python@v3 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - id: install_dependencies - run: | - python -m pip install --upgrade pip - pip install . - pip install -r test-requirements.txt - - - name: Run Task Runner E2E tests without TLS - id: run_tests - run: | - python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ - -m ${{ env.MODEL_NAME }} --model_name ${{ env.MODEL_NAME }} \ - --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_tls - echo "Task runner end to end test run completed" - - - name: Print test summary - id: print_test_summary - if: ${{ always() }} - run: | - export PYTHONPATH="$PYTHONPATH:." - python tests/end_to_end/utils/summary_helper.py - echo "Test summary printed" - - - name: Create Tar (exclude cert and data folders) - id: tar_files - if: ${{ always() }} - run: | - tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results - - - name: Upload Artifacts - id: upload_artifacts - uses: actions/upload-artifact@v4 - if: ${{ always() }} - with: - name: tr_non_tls_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} - path: result.tar - - test_with_no_client_auth: - name: tr_no_client_auth - runs-on: ubuntu-22.04 - timeout-minutes: 15 - strategy: - matrix: - # Testing non TLS scenario only for torch_cnn_mnist model and python 3.10 - # If required, this can be extended to other models and python versions - model_name: ["keras_cnn_mnist"] - python_version: ["3.10"] - fail-fast: false # do not immediately fail if one of the combinations fail - - env: - MODEL_NAME: ${{ matrix.model_name }} - PYTHON_VERSION: ${{ matrix.python_version }} - - steps: - - name: Checkout OpenFL repository - id: checkout_openfl - uses: actions/checkout@v4.1.1 - with: - fetch-depth: 2 # needed for detecting changes - submodules: "true" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Python - id: setup_python - uses: actions/setup-python@v3 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - id: install_dependencies - run: | - python -m pip install --upgrade pip - pip install . - pip install -r test-requirements.txt - - - name: Run Task Runner E2E tests without TLS - id: run_tests - run: | - python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ - -m ${{ env.MODEL_NAME }} --model_name ${{ env.MODEL_NAME }} \ - --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_client_auth - echo "Task runner end to end test run completed" - - - name: Print test summary - id: print_test_summary - if: ${{ always() }} - run: | - export PYTHONPATH="$PYTHONPATH:." - python tests/end_to_end/utils/summary_helper.py - echo "Test summary printed" - - - name: Create Tar (exclude cert and data folders) - id: tar_files - if: ${{ always() }} - run: | - tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results - - - name: Upload Artifacts - id: upload_artifacts - uses: actions/upload-artifact@v4 - if: ${{ always() }} - with: - name: tr_no_client_auth_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} - path: result.tar + # test_with_tls: + # name: tr_tls + # runs-on: ubuntu-22.04 + # timeout-minutes: 15 + # strategy: + # matrix: + # # There are open issues for some of the models, so excluding them for now: + # # model_name: [ "torch_cnn_mnist", "keras_cnn_mnist", "torch_cnn_histology" ] + # model_name: ["torch_cnn_mnist", "keras_cnn_mnist"] + # python_version: ["3.9", "3.10", "3.11"] + # fail-fast: false # do not immediately fail if one of the combinations fail + + # env: + # MODEL_NAME: ${{ matrix.model_name }} + # PYTHON_VERSION: ${{ matrix.python_version }} + + # steps: + # - name: Checkout OpenFL repository + # id: checkout_openfl + # uses: actions/checkout@v4.1.1 + # with: + # fetch-depth: 2 # needed for detecting changes + # submodules: "true" + # token: ${{ secrets.GITHUB_TOKEN }} + + # - name: Set up Python + # id: setup_python + # uses: actions/setup-python@v3 + # with: + # python-version: ${{ env.PYTHON_VERSION }} + + # - name: Install dependencies + # id: install_dependencies + # run: | + # python -m pip install --upgrade pip + # pip install . + # pip install -r test-requirements.txt + + # - name: Run Task Runner E2E tests with TLS + # id: run_tests + # run: | + # python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ + # -m ${{ env.MODEL_NAME }} --model_name ${{ env.MODEL_NAME }} \ + # --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} + # echo "Task runner end to end test run completed" + + # - name: Print test summary + # id: print_test_summary + # if: ${{ always() }} + # run: | + # export PYTHONPATH="$PYTHONPATH:." + # python tests/end_to_end/utils/summary_helper.py + # echo "Test summary printed" + + # - name: Create Tar (exclude cert and data folders) + # id: tar_files + # if: ${{ always() }} + # run: | + # tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results + + # - name: Upload Artifacts + # id: upload_artifacts + # uses: actions/upload-artifact@v4 + # if: ${{ always() }} + # with: + # name: tr_tls_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} + # path: result.tar + + # test_with_non_tls: + # name: tr_non_tls + # runs-on: ubuntu-22.04 + # timeout-minutes: 15 + # strategy: + # matrix: + # # Testing non TLS scenario only for torch_cnn_mnist model and python 3.10 + # # If required, this can be extended to other models and python versions + # model_name: ["torch_cnn_mnist"] + # python_version: ["3.10"] + # fail-fast: false # do not immediately fail if one of the combinations fail + + # env: + # MODEL_NAME: ${{ matrix.model_name }} + # PYTHON_VERSION: ${{ matrix.python_version }} + + # steps: + # - name: Checkout OpenFL repository + # id: checkout_openfl + # uses: actions/checkout@v4.1.1 + # with: + # fetch-depth: 2 # needed for detecting changes + # submodules: "true" + # token: ${{ secrets.GITHUB_TOKEN }} + + # - name: Set up Python + # id: setup_python + # uses: actions/setup-python@v3 + # with: + # python-version: ${{ env.PYTHON_VERSION }} + + # - name: Install dependencies + # id: install_dependencies + # run: | + # python -m pip install --upgrade pip + # pip install . + # pip install -r test-requirements.txt + + # - name: Run Task Runner E2E tests without TLS + # id: run_tests + # run: | + # python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ + # -m ${{ env.MODEL_NAME }} --model_name ${{ env.MODEL_NAME }} \ + # --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_tls + # echo "Task runner end to end test run completed" + + # - name: Print test summary + # id: print_test_summary + # if: ${{ always() }} + # run: | + # export PYTHONPATH="$PYTHONPATH:." + # python tests/end_to_end/utils/summary_helper.py + # echo "Test summary printed" + + # - name: Create Tar (exclude cert and data folders) + # id: tar_files + # if: ${{ always() }} + # run: | + # tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results + + # - name: Upload Artifacts + # id: upload_artifacts + # uses: actions/upload-artifact@v4 + # if: ${{ always() }} + # with: + # name: tr_non_tls_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} + # path: result.tar + + # test_with_no_client_auth: + # name: tr_no_client_auth + # runs-on: ubuntu-22.04 + # timeout-minutes: 15 + # strategy: + # matrix: + # # Testing non TLS scenario only for torch_cnn_mnist model and python 3.10 + # # If required, this can be extended to other models and python versions + # model_name: ["keras_cnn_mnist"] + # python_version: ["3.10"] + # fail-fast: false # do not immediately fail if one of the combinations fail + + # env: + # MODEL_NAME: ${{ matrix.model_name }} + # PYTHON_VERSION: ${{ matrix.python_version }} + + # steps: + # - name: Checkout OpenFL repository + # id: checkout_openfl + # uses: actions/checkout@v4.1.1 + # with: + # fetch-depth: 2 # needed for detecting changes + # submodules: "true" + # token: ${{ secrets.GITHUB_TOKEN }} + + # - name: Set up Python + # id: setup_python + # uses: actions/setup-python@v3 + # with: + # python-version: ${{ env.PYTHON_VERSION }} + + # - name: Install dependencies + # id: install_dependencies + # run: | + # python -m pip install --upgrade pip + # pip install . + # pip install -r test-requirements.txt + + # - name: Run Task Runner E2E tests without TLS + # id: run_tests + # run: | + # python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ + # -m ${{ env.MODEL_NAME }} --model_name ${{ env.MODEL_NAME }} \ + # --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_client_auth + # echo "Task runner end to end test run completed" + + # - name: Print test summary + # id: print_test_summary + # if: ${{ always() }} + # run: | + # export PYTHONPATH="$PYTHONPATH:." + # python tests/end_to_end/utils/summary_helper.py + # echo "Test summary printed" + + # - name: Create Tar (exclude cert and data folders) + # id: tar_files + # if: ${{ always() }} + # run: | + # tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results + + # - name: Upload Artifacts + # id: upload_artifacts + # uses: actions/upload-artifact@v4 + # if: ${{ always() }} + # with: + # name: tr_no_client_auth_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} + # path: result.tar test_memory_logs: name: tr_tls_memory_logs @@ -291,7 +291,7 @@ jobs: - name: Tar files id: tar_files if: ${{ always() }} - run: tar -cvf result.tar results + run: tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results - name: Upload Artifacts id: upload_artifacts diff --git a/tests/end_to_end/test_suites/memory_logs_tests.py b/tests/end_to_end/test_suites/memory_logs_tests.py index d763c419d0..b9692004c7 100644 --- a/tests/end_to_end/test_suites/memory_logs_tests.py +++ b/tests/end_to_end/test_suites/memory_logs_tests.py @@ -38,36 +38,55 @@ def test_log_memory_usage(request, fx_federation): # Setup PKI for trusted communication within the federation if request.config.use_tls: - assert fed_helper.setup_pki(fx_federation), "Failed to setup PKI for trusted communication" + assert fed_helper.setup_pki( + fx_federation + ), "Failed to setup PKI for trusted communication" # Start the federation results = fed_helper.run_federation(fx_federation) # Verify the completion of the federation run - assert fed_helper.verify_federation_run_completion(fx_federation, results, \ - num_rounds=request.config.num_rounds), "Federation completion failed" + assert fed_helper.verify_federation_run_completion( + fx_federation, results, num_rounds=request.config.num_rounds + ), "Federation completion failed" # Verify the aggregator memory logs - aggregator_memory_usage_file = os.path.join(fx_federation.workspace_path, "logs", "aggregator_memory_usage.json") - assert os.path.exists(aggregator_memory_usage_file), "Aggregator memory usage file is not available" + aggregator_memory_usage_file = os.path.join( + fx_federation.workspace_path, + "aggregator", + "workspace", + "logs", + "aggregator_memory_usage.json", + ) + assert os.path.exists( + aggregator_memory_usage_file + ), "Aggregator memory usage file is not available" # Log the aggregator memory usage details memory_usage_dict = json.load(open(aggregator_memory_usage_file)) # check memory usage entries for each round - assert len(memory_usage_dict) == request.config.num_rounds, \ - "Memory usage details are not available for all rounds" + assert ( + len(memory_usage_dict) == request.config.num_rounds + ), "Memory usage details are not available for all rounds" # check memory usage entries for each collaborator for collaborator in fx_federation.collaborators: - collaborator_memory_usage_file = os.path.join(fx_federation.workspace_path, - "logs", - f"{collaborator.collaborator_name}_memory_usage.json") + collaborator_memory_usage_file = os.path.join( + fx_federation.workspace_path, + collaborator.name, + "workspace", + "logs", + f"{collaborator.collaborator_name}_memory_usage.json", + ) - assert os.path.exists(collaborator_memory_usage_file), f"Memory usage file for collaborator {collaborator.collaborator_name} is not available" + assert os.path.exists( + collaborator_memory_usage_file + ), f"Memory usage file for collaborator {collaborator.collaborator_name} is not available" memory_usage_dict = json.load(open(collaborator_memory_usage_file)) - assert len(memory_usage_dict) == request.config.num_rounds, \ - f"Memory usage details are not available for all rounds for collaborator {collaborator.collaborator_name}" + assert ( + len(memory_usage_dict) == request.config.num_rounds + ), f"Memory usage details are not available for all rounds for collaborator {collaborator.collaborator_name}" log.info("Memory usage details are available for all participants")