task_runner_basic_e2e.yml

---
# Task Runner E2E tests for bare metal approach

name: Task_Runner_E2E  # Please do not modify the name as it is used in the composite action

on:
  schedule:
    - cron: "0 0 * * *" # Run every day at midnight
  workflow_dispatch:
    inputs:
      num_rounds:
        description: "Number of rounds to train"
        required: false
        default: "5"
        type: string
      num_collaborators:
        description: "Number of collaborators"
        required: false
        default: "2"
        type: string

permissions:
  contents: read

# Environment variables common for all the jobs
env:
  NUM_ROUNDS: ${{ inputs.num_rounds || '5' }}
  NUM_COLLABORATORS: ${{ inputs.num_collaborators || '2' }}

jobs:
  test_with_tls:
    name: tr_tls
    runs-on: ubuntu-22.04
    timeout-minutes: 15
    strategy:
      matrix:
        # There are open issues for some of the models, so excluding them for now:
        # model_name: [ "torch_cnn_mnist", "keras_cnn_mnist", "torch_cnn_histology" ]
        model_name: ["torch_cnn_mnist", "keras_cnn_mnist"]
        python_version: ["3.10", "3.11", "3.12"]
      fail-fast: false # do not immediately fail if one of the combinations fail

    env:
      MODEL_NAME: ${{ matrix.model_name }}
      PYTHON_VERSION: ${{ matrix.python_version }}

    steps:
      - name: Checkout OpenFL repository
        id: checkout_openfl
        uses: actions/checkout@v4.1.1
        with:
          fetch-depth: 2 # needed for detecting changes
          submodules: "true"
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Pre test run
        uses: ./.github/actions/tr_pre_test_run
        if: ${{ always() }}

      - name: Run Task Runner E2E tests with TLS
        id: run_tests
        run: |
          python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \
          -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \
          --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }}
          echo "Task runner end to end test run completed"

      - name: Post test run
        uses: ./.github/actions/tr_post_test_run
        if: ${{ always() }}
        with:
          test_type: "tr_tls"

  test_with_non_tls:
    name: tr_non_tls
    runs-on: ubuntu-22.04
    timeout-minutes: 15
    strategy:
      matrix:
        # Testing non TLS scenario only for torch_cnn_mnist model and python 3.10
        # If required, this can be extended to other models and python versions
        model_name: ["torch_cnn_mnist"]
        python_version: ["3.10"]
      fail-fast: false # do not immediately fail if one of the combinations fail

    env:
      MODEL_NAME: ${{ matrix.model_name }}
      PYTHON_VERSION: ${{ matrix.python_version }}

    steps:
      - name: Checkout OpenFL repository
        id: checkout_openfl
        uses: actions/checkout@v4.1.1
        with:
          fetch-depth: 2 # needed for detecting changes
          submodules: "true"
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Pre test run
        uses: ./.github/actions/tr_pre_test_run
        if: ${{ always() }}

      - name: Run Task Runner E2E tests without TLS
        id: run_tests
        run: |
          python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \
          -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \
          --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_tls
          echo "Task runner end to end test run completed"

      - name: Post test run
        uses: ./.github/actions/tr_post_test_run
        if: ${{ always() }}
        with:
          test_type: "tr_non_tls"

  test_with_no_client_auth:
    name: tr_no_client_auth
    runs-on: ubuntu-22.04
    timeout-minutes: 15
    strategy:
      matrix:
        # Testing non TLS scenario only for torch_cnn_mnist model and python 3.10
        # If required, this can be extended to other models and python versions
        model_name: ["keras_cnn_mnist"]
        python_version: ["3.10"]
      fail-fast: false # do not immediately fail if one of the combinations fail

    env:
      MODEL_NAME: ${{ matrix.model_name }}
      PYTHON_VERSION: ${{ matrix.python_version }}

    steps:
      - name: Checkout OpenFL repository
        id: checkout_openfl
        uses: actions/checkout@v4.1.1
        with:
          fetch-depth: 2 # needed for detecting changes
          submodules: "true"
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Pre test run
        uses: ./.github/actions/tr_pre_test_run
        if: ${{ always() }}

      - name: Run Task Runner E2E tests without TLS
        id: run_tests
        run: |
          python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \
          -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \
          --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_client_auth
          echo "Task runner end to end test run completed"

      - name: Post test run
        uses: ./.github/actions/tr_post_test_run
        if: ${{ always() }}
        with:
          test_type: "tr_no_client_auth"

  test_memory_logs:
    name: tr_tls_memory_logs
    runs-on: ubuntu-22.04
    timeout-minutes: 15
    strategy:
      matrix:
        # Testing non TLS scenario only for torch_cnn_mnist model and python 3.10
        # If required, this can be extended to other models and python versions
        model_name: ["torch_cnn_mnist"]
        python_version: ["3.10"]
      fail-fast: false # do not immediately fail if one of the combinations fail

    env:
      MODEL_NAME: ${{ matrix.model_name }}
      PYTHON_VERSION: ${{ matrix.python_version }}

    steps:
      - name: Checkout OpenFL repository
        id: checkout_openfl
        uses: actions/checkout@v4.1.1
        with:
          fetch-depth: 2 # needed for detecting changes
          submodules: "true"
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Pre test run
        uses: ./.github/actions/tr_pre_test_run
        if: ${{ always() }}

      - name: Run Task Runner E2E tests with TLS and memory logs
        id: run_tests
        run: |
          python -m pytest -s tests/end_to_end/test_suites/memory_logs_tests.py \
          -k test_log_memory_usage_basic --model_name ${{ env.MODEL_NAME }} \
          --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} \
          --log_memory_usage
          echo "Task runner memory logs test run completed"

      - name: Post test run
        uses: ./.github/actions/tr_post_test_run
        if: ${{ always() }}
        with:
          test_type: "tr_tls_memory_logs"