diff --git a/.github/workflows/task_runner_docker_e2e.yml b/.github/workflows/task_runner_docker_e2e.yml index 82c196456a..b3953503ed 100644 --- a/.github/workflows/task_runner_docker_e2e.yml +++ b/.github/workflows/task_runner_docker_e2e.yml @@ -28,80 +28,81 @@ env: NUM_COLLABORATORS: ${{ inputs.num_collaborators || '2' }} jobs: - test_with_tls_docker: - name: tr_tls_docker - runs-on: ubuntu-22.04 - timeout-minutes: 15 - strategy: - matrix: - # There are open issues for some of the models, so excluding them for now: - # model_name: [ "torch_cnn_mnist", "keras_cnn_mnist", "torch_cnn_histology" ] - model_name: ["keras_cnn_mnist"] - python_version: ["3.9"] - fail-fast: false # do not immediately fail if one of the combinations fail - - env: - MODEL_NAME: ${{ matrix.model_name }} - PYTHON_VERSION: ${{ matrix.python_version }} - - steps: - - name: Checkout OpenFL repository - id: checkout_openfl - uses: actions/checkout@v4.1.1 - with: - fetch-depth: 2 # needed for detecting changes - submodules: "true" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Python - id: setup_python - uses: actions/setup-python@v3 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - id: install_dependencies - run: | - python -m pip install --upgrade pip - pip install . - pip install -r test-requirements.txt - - - name: Create OpenFL image - id: create_openfl_image - run: | - echo "Creating openfl image. This may take a few minutes..." - cd openfl-docker - docker build . -t openfl -f Dockerfile.base - - - name: Run Task Runner E2E tests with TLS - id: run_tests - run: | - python -m pytest -s tests/end_to_end/test_suites/docker_tests.py \ - -m docker --model_name ${{ env.MODEL_NAME }} \ - --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} - echo "Task runner end to end test run completed" - - - name: Print test summary - id: print_test_summary - if: ${{ always() }} - run: | - export PYTHONPATH="$PYTHONPATH:." - python tests/end_to_end/utils/summary_helper.py - echo "Test summary printed" - - - name: Create Tar (exclude cert and data folders) - id: tar_files - if: ${{ always() }} - run: | - tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results - - - name: Upload Artifacts - id: upload_artifacts - uses: actions/upload-artifact@v4 - if: ${{ always() }} - with: - name: tr_tls_docker_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} - path: result.tar + # test_with_tls_docker: + # name: tr_tls_docker + # runs-on: ubuntu-22.04 + # timeout-minutes: 15 + # strategy: + # matrix: + # # There are open issues for some of the models, so excluding them for now: + # # model_name: [ "torch_cnn_mnist", "keras_cnn_mnist", "torch_cnn_histology" ] + # model_name: ["keras_cnn_mnist"] + # python_version: ["3.9"] + # fail-fast: false # do not immediately fail if one of the combinations fail + + # env: + # MODEL_NAME: ${{ matrix.model_name }} + # PYTHON_VERSION: ${{ matrix.python_version }} + + # steps: + # - name: Checkout OpenFL repository + # id: checkout_openfl + # uses: actions/checkout@v4.1.1 + # with: + # fetch-depth: 2 # needed for detecting changes + # submodules: "true" + # token: ${{ secrets.GITHUB_TOKEN }} + + # - name: Set up Python + # id: setup_python + # uses: actions/setup-python@v3 + # with: + # python-version: ${{ env.PYTHON_VERSION }} + + # - name: Install dependencies + # id: install_dependencies + # run: | + # python -m pip install --upgrade pip + # pip install . + # pip install -r test-requirements.txt + + # - name: Create OpenFL image + # id: create_openfl_image + # run: | + # echo "Creating openfl image. This may take a few minutes..." + # cd openfl-docker + # docker build . -t openfl -f Dockerfile.base \ + # --build-arg OPENFL_REVISION=https://github.com/securefederatedai/openfl.git@develop + + # - name: Run Task Runner E2E tests with TLS + # id: run_tests + # run: | + # python -m pytest -s tests/end_to_end/test_suites/docker_tests.py \ + # -m docker --model_name ${{ env.MODEL_NAME }} \ + # --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} + # echo "Task runner end to end test run completed" + + # - name: Print test summary + # id: print_test_summary + # if: ${{ always() }} + # run: | + # export PYTHONPATH="$PYTHONPATH:." + # python tests/end_to_end/utils/summary_helper.py + # echo "Test summary printed" + + # - name: Create Tar (exclude cert and data folders) + # id: tar_files + # if: ${{ always() }} + # run: | + # tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results + + # - name: Upload Artifacts + # id: upload_artifacts + # uses: actions/upload-artifact@v4 + # if: ${{ always() }} + # with: + # name: tr_tls_docker_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} + # path: result.tar test_with_non_tls_docker: name: tr_non_tls_docker @@ -128,127 +129,131 @@ jobs: submodules: "true" token: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Python - id: setup_python - uses: actions/setup-python@v3 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - id: install_dependencies - run: | - python -m pip install --upgrade pip - pip install . - pip install -r test-requirements.txt - - - name: Create OpenFL image - id: create_openfl_image - run: | - echo "Creating openfl image. This may take a few minutes..." - cd openfl-docker - docker build . -t openfl -f Dockerfile.base - - - name: Run Task Runner E2E tests without TLS - id: run_tests - run: | - python -m pytest -s tests/end_to_end/test_suites/docker_tests.py \ - -m docker --model_name ${{ env.MODEL_NAME }} \ - --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_tls - echo "Task runner end to end test run completed" - - - name: Print test summary - id: print_test_summary - if: ${{ always() }} - run: | - export PYTHONPATH="$PYTHONPATH:." - python tests/end_to_end/utils/summary_helper.py - echo "Test summary printed" - - - name: Create Tar (exclude cert and data folders) - id: tar_files - if: ${{ always() }} - run: | - tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results - - - name: Upload Artifacts - id: upload_artifacts - uses: actions/upload-artifact@v4 - if: ${{ always() }} - with: - name: tr_non_tls_docker_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} - path: result.tar - - test_with_no_client_auth_docker: - name: tr_no_client_auth_docker - runs-on: ubuntu-22.04 - timeout-minutes: 15 - strategy: - matrix: - # Testing non TLS scenario only for keras_cnn_mnist model and python 3.10 - # If required, this can be extended to other models and python versions - model_name: ["keras_cnn_mnist"] - python_version: ["3.11"] - fail-fast: false # do not immediately fail if one of the combinations fail - - env: - MODEL_NAME: ${{ matrix.model_name }} - PYTHON_VERSION: ${{ matrix.python_version }} - - steps: - - name: Checkout OpenFL repository - id: checkout_openfl - uses: actions/checkout@v4.1.1 - with: - fetch-depth: 2 # needed for detecting changes - submodules: "true" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Python - id: setup_python - uses: actions/setup-python@v3 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - id: install_dependencies + - name: Print GitHub variables run: | - python -m pip install --upgrade pip - pip install . - pip install -r test-requirements.txt - - - name: Create OpenFL image - id: create_openfl_image - run: | - echo "Creating openfl image. This may take a few minutes..." - cd openfl-docker - docker build . -t openfl -f Dockerfile.base - - - name: Run Task Runner E2E tests without TLS - id: run_tests - run: | - python -m pytest -s tests/end_to_end/test_suites/docker_tests.py \ - -m docker --model_name ${{ env.MODEL_NAME }} \ - --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_client_auth - echo "Task runner end to end test run completed" - - - name: Print test summary - id: print_test_summary - if: ${{ always() }} - run: | - export PYTHONPATH="$PYTHONPATH:." - python tests/end_to_end/utils/summary_helper.py - echo "Test summary printed" - - - name: Create Tar (exclude cert and data folders) - id: tar_files - if: ${{ always() }} - run: | - tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results - - - name: Upload Artifacts - id: upload_artifacts - uses: actions/upload-artifact@v4 - if: ${{ always() }} - with: - name: tr_no_client_auth_docker_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} - path: result.tar + echo "${{ toJson(github) }}" + + # - name: Set up Python + # id: setup_python + # uses: actions/setup-python@v3 + # with: + # python-version: ${{ env.PYTHON_VERSION }} + + # - name: Install dependencies + # id: install_dependencies + # run: | + # python -m pip install --upgrade pip + # pip install . + # pip install -r test-requirements.txt + + # - name: Create OpenFL image + # id: create_openfl_image + # run: | + # echo "Creating openfl image. This may take a few minutes..." + # cd openfl-docker + # docker build . -t openfl -f Dockerfile.base + + # - name: Run Task Runner E2E tests without TLS + # id: run_tests + # run: | + # python -m pytest -s tests/end_to_end/test_suites/docker_tests.py \ + # -m docker --model_name ${{ env.MODEL_NAME }} \ + # --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_tls + # echo "Task runner end to end test run completed" + + # - name: Print test summary + # id: print_test_summary + # if: ${{ always() }} + # run: | + # export PYTHONPATH="$PYTHONPATH:." + # python tests/end_to_end/utils/summary_helper.py + # echo "Test summary printed" + + # - name: Create Tar (exclude cert and data folders) + # id: tar_files + # if: ${{ always() }} + # run: | + # tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results + + # - name: Upload Artifacts + # id: upload_artifacts + # uses: actions/upload-artifact@v4 + # if: ${{ always() }} + # with: + # name: tr_non_tls_docker_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} + # path: result.tar + + # test_with_no_client_auth_docker: + # name: tr_no_client_auth_docker + # runs-on: ubuntu-22.04 + # timeout-minutes: 15 + # strategy: + # matrix: + # # Testing non TLS scenario only for keras_cnn_mnist model and python 3.10 + # # If required, this can be extended to other models and python versions + # model_name: ["keras_cnn_mnist"] + # python_version: ["3.11"] + # fail-fast: false # do not immediately fail if one of the combinations fail + + # env: + # MODEL_NAME: ${{ matrix.model_name }} + # PYTHON_VERSION: ${{ matrix.python_version }} + + # steps: + # - name: Checkout OpenFL repository + # id: checkout_openfl + # uses: actions/checkout@v4.1.1 + # with: + # fetch-depth: 2 # needed for detecting changes + # submodules: "true" + # token: ${{ secrets.GITHUB_TOKEN }} + + # - name: Set up Python + # id: setup_python + # uses: actions/setup-python@v3 + # with: + # python-version: ${{ env.PYTHON_VERSION }} + + # - name: Install dependencies + # id: install_dependencies + # run: | + # python -m pip install --upgrade pip + # pip install . + # pip install -r test-requirements.txt + + # - name: Create OpenFL image + # id: create_openfl_image + # run: | + # echo "Creating openfl image. This may take a few minutes..." + # cd openfl-docker + # docker build . -t openfl -f Dockerfile.base + + # - name: Run Task Runner E2E tests without TLS + # id: run_tests + # run: | + # python -m pytest -s tests/end_to_end/test_suites/docker_tests.py \ + # -m docker --model_name ${{ env.MODEL_NAME }} \ + # --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_client_auth + # echo "Task runner end to end test run completed" + + # - name: Print test summary + # id: print_test_summary + # if: ${{ always() }} + # run: | + # export PYTHONPATH="$PYTHONPATH:." + # python tests/end_to_end/utils/summary_helper.py + # echo "Test summary printed" + + # - name: Create Tar (exclude cert and data folders) + # id: tar_files + # if: ${{ always() }} + # run: | + # tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" $HOME/results + + # - name: Upload Artifacts + # id: upload_artifacts + # uses: actions/upload-artifact@v4 + # if: ${{ always() }} + # with: + # name: tr_no_client_auth_docker_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }} + # path: result.tar