run_tests.yml

name: Run Tests

on:
  push:
    tags:
      - '*'
    branches:
      - '**'
  pull_request:
    types: [opened, synchronize, reopened, ready_for_review]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build_images:

    runs-on: ubuntu-latest
    if: |
      !endsWith(github.event.head_commit.message, '# ci skip') &&
      !startsWith(github.ref, 'refs/heads/test_process') &&
      (
        github.event_name != 'pull_request' ||
        (
          github.event_name == 'pull_request' &&
          github.event.pull_request.draft == false &&
          github.event.pull_request.head.repo.owner.id == github.event.pull_request.base.repo.owner.id
        )
      )

    env:
      BRANCH_NAME: "auto_update_docker_${{ github.run_number }}"

    outputs:
      images: ${{ steps.export-images.outputs.images }}

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 1000

    - name: Clear space on runner
      run: ./scripts/clear_runner_diskspace.sh

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: "3.8"

    - name: Set up environment
      run: |
        echo "LINUX_VERSION=$(uname -rs)" >> $GITHUB_ENV

    - name: Cache Python packages
      uses: actions/cache@v3
      with:
        path: ${{ env.pythonLocation }}
        key: python-pip-${{ env.LINUX_VERSION }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}
        restore-keys: python-pip-${{ env.LINUX_VERSION }}-${{ env.pythonLocation }}-

    - name: Install package & dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -U wheel setuptools
        pip install --editable .[evaluate]
        python -c "import openproblems"

    - name: Update Docker images
      if: |
        !(
          startsWith(github.ref, 'refs/heads/test_docker') ||
          startsWith(github.ref, 'refs/heads/main')
        )
      run: |
        cd workflow
        SNAKEMAKE_COPY_SOURCE=1 snakemake -j $(nproc) docker
        cd ..

    - name: Build Docker images
      if: |
        startsWith(github.ref, 'refs/heads/test_docker')
      run: |
        cd workflow
        snakemake -j $(nproc) docker_build
        cd ..

    - name: Build and push Docker images
      if: |
        startsWith(github.ref, 'refs/heads/main')
      env:
        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
      run: |
        cd workflow
        snakemake -j $(nproc) docker_build docker_push
        cd ..
        echo "CHANGED=`git diff --exit-code > /dev/null && echo false || echo true`" >> $GITHUB_ENV

    - name: Create Pull Request
      if: env.CHANGED == 'true' && startsWith(github.ref, 'refs/heads/main')
      uses: peter-evans/create-pull-request@v4
      with:
        branch: ${{ env.BRANCH_NAME }}
        delete-branch: true
        base: main
        title: '[auto] Update docker version'
        reviewers: scottgigante, dburkhardt
        author: "openproblems-bio <singlecellopenproblems@protonmail.com>"
        commit-message: "Update docker version # ci skip"
        add-paths: docker/.version

    - name: Log in to the Container registry
      uses: docker/login-action@v2
      with:
        registry: ghcr.io
        username: ${{ github.repository_owner }}
        password: ${{ secrets.GITHUB_TOKEN }}

    - name: Export docker images
      id: export-images
      run: |
        IMAGES="$(find ./docker -mindepth 1 -type d -exec basename {} \;)"
        for image in ${IMAGES}; do
          GHCR_IMAGE="ghcr.io/${{ github.repository_owner }}/${image}:${{ github.run_id }}"
          docker tag singlecellopenproblems/$image $GHCR_IMAGE
          docker push $GHCR_IMAGE &
          PIDS+=("${!}")
        done
        for pid in "${PIDS[@]}"; do
          # ensure exited with status 0
          wait $pid
        done
        # convert to JSON
        echo "images=[\"$(paste -s -d ' ' <(echo $IMAGES) | sed 's/  */\",\"/g')\"]" >> $GITHUB_OUTPUT
      shell: bash -e {0}

  create_matrix:
    runs-on: ubuntu-latest
    if: |
      !endsWith(github.event.head_commit.message, '# ci skip') &&
      !startsWith(github.ref, 'refs/heads/test_docker') &&
      !startsWith(github.ref, 'refs/heads/test_benchmark') &&
      !startsWith(github.ref, 'refs/heads/test_full_benchmark') &&
      !startsWith(github.ref, 'refs/heads/test_process') &&
      !startsWith(github.ref, 'refs/heads/test_website') &&
      (
        github.event_name != 'pull_request' ||
        (
          github.event_name == 'pull_request' &&
          github.event.pull_request.draft == false &&
          github.actor != 'dependabot[bot]'
        )
      )

    outputs:
      matrix: ${{ steps.generate-matrix.outputs.matrix }}

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 1

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: "3.8"

    - name: Set up environment
      run: |
        echo "LINUX_VERSION=$(uname -rs)" >> $GITHUB_ENV

    - name: Cache Python packages
      uses: actions/cache@v3
      with:
        path: ${{ env.pythonLocation }}
        key: python-pip-${{ env.LINUX_VERSION }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}
        restore-keys: python-pip-${{ env.LINUX_VERSION }}-${{ env.pythonLocation }}-

    - name: Install package & dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -U wheel setuptools
        pip install --editable .[evaluate]
        python -c "import openproblems"


    - name: Create test matrix
      id: generate-matrix
      run: |
        set -eo pipefail
        echo "matrix=$(python scripts/generate_test_matrix.py)" >> $GITHUB_OUTPUT

  run_tester:
    runs-on: ubuntu-latest

    container:
      image: singlecellopenproblems/openproblems-github-actions:latest
      volumes:
        - /var/run/docker.sock:/var/run/docker.sock
        - /tmp:/tmp
        - /opt/az:/opt/remove/az
        - /usr/lib/google-cloud-sdk:/opt/remove/google-cloud-sdk
        - /usr/lib/jvm:/opt/remove/jvm
        - /opt/google/chrome:/opt/remove/chrome
        - /usr/lib/firefox:/opt/remove/firefox
        - /opt/microsoft/powershell:/opt/remove/powershell
        - /usr/share/dotnet:/opt/remove/dotnet
        - /opt/ghc:/opt/remove/ghc
        - /opt/hostedtoolcache:/opt/remove/hostedtoolcache
        - /usr/local/lib/android:/opt/remove/android
      options: --user root

    needs:
      - build_images
      - create_matrix
    if: |
      always() &&
      !endsWith(github.event.head_commit.message, '# ci skip') &&
      needs.create_matrix.result == 'success' &&
      (
        needs.build_images.result == 'success' ||
        needs.build_images.result == 'skipped'
      )

    strategy:
      fail-fast: false
      matrix:
        tests: ${{ fromJSON(needs.create_matrix.outputs.matrix) }}

    steps:
    - name: Clear space on runner
      run: |
        sudo find /opt/remove -mindepth 2 -maxdepth 2 -type d -exec rm -rf {} \;

    - uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - name: Log in to the Container registry
      uses: docker/login-action@v2
      if: "needs.build_images.result == 'success'"
      with:
        registry: ghcr.io
        username: ${{ github.repository_owner }}
        password: ${{ secrets.GITHUB_TOKEN }}

    - name: Download docker images
      if: "needs.build_images.result == 'success'"
      env:
        REPO_OWNER: ${{ github.repository_owner }}
        RUN_ID: ${{ github.run_id }}
      run: |
        for image in $(find ./docker -mindepth 1 -type d -exec basename {} \;); do
          GHCR_IMAGE="ghcr.io/${REPO_OWNER}/${image}:${RUN_ID}"
          docker pull $GHCR_IMAGE &
          PIDS+=("${!}")
        done
        for pid in "${PIDS[@]}"; do
          # ensure exited with status 0
          wait $pid
        done
        # tag images
        for image in $(find ./docker -mindepth 1 -type d -exec basename {} \;); do
          GHCR_IMAGE="ghcr.io/${REPO_OWNER}/${image}:${RUN_ID}"
          docker tag $GHCR_IMAGE singlecellopenproblems/${image}:latest
        done
      shell: bash -e {0}

    - name: Set up environment
      run: |
        echo "LINUX_VERSION=$(uname -rs)" >> $GITHUB_ENV
        echo "pythonLocation=$(which python)" >> $GITHUB_ENV
        echo "PYTHON_VERSION=$(python --version)" >> $GITHUB_ENV
        echo "R_VERSION=$(R --version | head -n 1)" >> $GITHUB_ENV

    - name: Cache Python packages
      uses: actions/cache@v3
      with:
        path: ${{ env.pythonLocation }}
        key: python-pip-${{env.LINUX_VERSION}}-pip-${{ env.PYTHON_VERSION }}-${{ hashFiles('setup.py') }}
        restore-keys: python-pip-${{env.LINUX_VERSION}}-pip-${{ env.PYTHON_VERSION }}-

    - name: Install package & dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -U wheel setuptools
        pip install -U --editable .[test,r,evaluate]
        python -c "import openproblems"

    - name: Cache R packages
      uses: actions/cache@v3
      if: startsWith(runner.os, 'Linux')
      with:
        path: /usr/local/lib/R/site-library
        key: ${{ env.LINUX_VERSION }}-renv-${{ env.R_VERSION }}-${{ hashFiles('**/renv.lock') }}
        restore-keys: |
          ${{ env.LINUX_VERSION }}-renv-${{ env.R_VERSION }}-

    - name: Install R packages
      env:
        BIOCVERSION: '3.16'
      run: |
        if (!requireNamespace("renv", quietly = TRUE)) install.packages("renv")
        renv::restore()
        source("./scripts/install_renv.R")
        install_renv("docker/openproblems-r-base/r_requirements.txt")
        install_renv("docker/openproblems-github-actions/r_requirements.txt")
      shell: Rscript {0}

    - name: Update Docker images
      if: "needs.build_images.result == 'skipped'"
      run: |
        cd workflow
        snakemake -j $(nproc) docker
        cd ..

    - name: Run tests
      timeout-minutes: 60
      run: |
        pytest --cov=openproblems --cov-report=xml -vv --durations=15 --tb=native -k "${{ matrix.tests }}"
        mkdir -p coverage
        mv coverage.xml "$(echo 'coverage_${{ matrix.tests }}.xml' | sed 's/[^a-z0-9\.]/_/g')"

    - name: Upload coverage to GitHub Actions
      uses: actions/upload-artifact@main
      with:
        path: coverage_*.xml
        name: coverage

  upload_coverage:
    needs: run_tester
    runs-on: ubuntu-latest
    if: >-
      always() &&
      !endsWith(github.event.head_commit.message, '# ci skip') &&
      needs.run_tester.result == 'success'

    steps:

    - uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - name: Download coverage from GitHub Actions
      uses: actions/download-artifact@v3
      with:
        name: coverage

    - name: Upload coverage
      uses: codecov/codecov-action@v3
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        flags: unittests
        fail_ci_if_error: ${{ github.repository == 'openproblems-bio/openproblems' }}
        verbose: true

    - name: Delete coverage artifacts
      uses: geekyeggo/delete-artifact@v2
      with:
          name: coverage

  setup_benchmark:
    needs:
    - run_tester
    - build_images
    runs-on: ubuntu-latest
    if: >-
      always() &&
      !endsWith(github.event.head_commit.message, '# ci skip') &&
      needs.build_images.result == 'success' &&
      github.event_name != 'pull_request' &&
      (
        needs.run_tester.result == 'success' ||
        needs.run_tester.result == 'skipped'
      )

    outputs:
      branch: ${{ steps.setup-environment.outputs.branch }}
      run_name: ${{ steps.setup-environment.outputs.run_name }}

    steps:
    - name: Check dependabot
      run: |
        if [[ "${{ github.actor }}" == "dependabot[bot]" ]]; then
          echo "Dependabot cannot access Action secrets. Rerun the workflow manually"
          exit 1
        fi

    - name: Check Tower authentication
      env:
        TOWER_ACCESS_TOKEN: ${{ secrets.TOWER_ACCESS_KEY }}
        TOWER_ORG_ID: 9180144064527
        TOWER_WORKSPACE_ID: 53907369739130
      run: |
        token_confirmation=$(
          curl -X GET "https://api.tower.nf/user-info" \
          -H "Accept: application/json" \
          -H "Authorization: Bearer ${TOWER_ACCESS_TOKEN}"
        )
        if [ -z "${token_confirmation}" ]; then
          echo "Tower authentication failed; set TOWER_ACCESS_KEY in repo secrets."
          echo "See CONTRIBUTING.md for details."
          exit 1
        fi
        workspace_confirmation=$(
          curl -X GET "https://api.tower.nf/orgs/${TOWER_ORG_ID}/workspaces/${TOWER_WORKSPACE_ID}" \
          -H "Accept: application/json" \
          -H "Authorization: Bearer ${TOWER_ACCESS_TOKEN}"
        )
        if [ -z "${workspace_confirmation}" ]; then
          echo "Tower authentication failed; email singlecellopenproblems@protonmail.com with a request to be added to the openproblems-bio workspace"
          exit 1
        fi

    - uses: actions/checkout@v3
      with:
        fetch-depth: 1

    - name: Clear space on runner
      run: ./scripts/clear_runner_diskspace.sh

    - name: Log in to the Container registry
      uses: docker/login-action@v2
      with:
        registry: ghcr.io
        username: ${{ github.repository_owner }}
        password: ${{ secrets.GITHUB_TOKEN }}

    - name: Download docker images
      env:
        REPO_OWNER: ${{ github.repository_owner }}
        RUN_ID: ${{ github.run_id }}
      run: |
        for image in $(find ./docker -mindepth 1 -type d -exec basename {} \;); do
          GHCR_IMAGE="ghcr.io/${REPO_OWNER}/${image}:${RUN_ID}"
          docker pull $GHCR_IMAGE &
          PIDS+=("${!}")
        done
        for pid in "${PIDS[@]}"; do
          # ensure exited with status 0
          wait $pid
        done
        # tag images
        for image in $(find ./docker -mindepth 1 -type d -exec basename {} \;); do
          GHCR_IMAGE="ghcr.io/${REPO_OWNER}/${image}:${RUN_ID}"
          docker tag $GHCR_IMAGE singlecellopenproblems/${image}:latest
        done
      shell: bash -e {0}

    - name: Set up environment
      id: setup-environment
      run: |
        # If not on the base repository, append first 6 characters of username to the image name
        # to avoid clashes on ECR
        REPO_PARSED=$(echo ${{ github.repository }} | awk '{print $1}' FS=/ | head -c 6)
        BRANCH_PARSED=$(echo ${{ github.ref }} | sed 's:refs/[a-z]*/::' | sed 's:[^a-zA-Z0-9]:-:g')
        if [[ ${{ startsWith(github.ref, 'refs/tags') || startsWith(github.ref, 'refs/heads/test_full_benchmark') }} == true ]]; then
          BRANCH="prod"
        elif [[ "${{ github.repository }}" == "openproblems-bio/openproblems" ]]; then
          BRANCH=`echo $BRANCH_PARSED | head -c 40`
        else
          BRANCH="${REPO_PARSED}-`echo $BRANCH_PARSED | head -c 33`"
        fi
        BRANCH=`echo $BRANCH | sed 's/[^a-zA-Z0-9]*$//'`
        echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
        echo "branch=${BRANCH}" >> $GITHUB_OUTPUT
        RUN_NAME="$(echo "$BRANCH" | sed "s/[^a-zA-Z0-9]/_/g")_$(git rev-parse --short HEAD)_${GITHUB_RUN_ATTEMPT}"
        echo "run_name=${RUN_NAME}" >> $GITHUB_OUTPUT

    - name: Upload Docker images
      env:
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        AWS_DEFAULT_REGION: us-west-2
      run: |
        ECR_ENDPOINT="490915662541.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com"
        aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | \
          docker login --username AWS --password-stdin $ECR_ENDPOINT
        for image in $(cd docker && ls -1d */ | tr -d '/'); do
          docker tag singlecellopenproblems/${image} ${ECR_ENDPOINT}/openproblems:${BRANCH}-${image}
          docker push --quiet ${ECR_ENDPOINT}/openproblems:${BRANCH}-${image} &
          PIDS+=("${!}")
        done
        for pid in "${PIDS[@]}"; do
          # ensure exited with status 0
          wait $pid
        done
      shell: bash -e {0}

  run_benchmark:
    needs:
    - setup_benchmark
    runs-on: ubuntu-latest
    if: >-
      always() &&
      needs.setup_benchmark.result == 'success'

    steps:
    - name: Run benchmark
      env:
        TOWER_WATCH_URL: https://tower.nf/orgs/openproblems-bio/workspaces/openproblems-bio/watch
        TOWER_WORKSPACE_ID: 53907369739130
        BRANCH: ${{ needs.setup_benchmark.outputs.branch }}
      run: |
        if [[ ${{ startsWith(github.ref, 'refs/tags') || startsWith(github.ref, 'refs/heads/test_full_benchmark') }} == true ]]; then
          TOWER_ACTION_ID="bVQhVSNah1JmJfnKkfyjg"
          WORKDIR="s3://openproblems-nextflow/work_main"
        else
          TOWER_ACTION_ID="5BQc88ZvjuXCYbc55Hot27"
          WORKDIR="s3://openproblems-nextflow/work/${BRANCH}"
        fi
        generate_parameters()
        {
        cat <<EOF
          {
            "params" : {
              "branch": "${BRANCH}"
            }
          }
        EOF
        }
        OUTPUT=$(
            curl -H "Content-Type: application/json" \
            -H "Authorization: Bearer ${{ secrets.TOWER_ACCESS_KEY }}" \
            https://api.tower.nf/actions/${TOWER_ACTION_ID}/launch?workspaceId=${TOWER_WORKSPACE_ID} \
            --data "$(generate_parameters)"
        )
        echo "$OUTPUT"
        if [ $(echo "$OUTPUT" | grep "No more than 5 workflow runs at time are allowed") ]; then
          echo "Nextflow Tower is currently rate limited. Please wait until fewer jobs are running and then retry failed jobs."
          exit 1
        fi
        WORKFLOW_ID=$(echo "$OUTPUT" | sed 's/.*"\([a-zA-Z0-9]*\)".*/\1/')
        echo "Benchmark running at ${TOWER_WATCH_URL}/${WORKFLOW_ID}" >> $GITHUB_STEP_SUMMARY