Refactored cache key logic some more #203
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: main | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| test_targets: | |
| default: "" | |
| description: "Comma separated list to run pytest on, e.g.: `lib/dl_api_lib,lib/dl_core`. " | |
| run_mypy_only: | |
| type: boolean | |
| default: false | |
| description: "Check to only run mypy" | |
| pytest_timeout_minutes: | |
| type: number | |
| default: 10 | |
| description: "Timeout for pytest JOB in minutes" | |
| jobs: | |
| gh_build_image: | |
| runs-on: [ self-hosted, linux ] | |
| permissions: | |
| packages: write | |
| contents: read | |
| container: | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/debian_docker:latest" | |
| options: -v /var/run/docker.sock:/var/run/docker.sock | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| env: | |
| REPO_OWNER: ${{ github.repository_owner }} | |
| REPO_NAME: ${{ github.event.repository.name }} | |
| GIT_SHA: "${{ github.sha }}" | |
| steps: | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: "ghcr.io" | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: 'Cleanup build folder' | |
| run: | | |
| rm -rf ./* || true | |
| rm -rf ./.??* || true | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} | |
| - run: | | |
| export ROOT_DIR="$(realpath .)" | |
| /bin/bash ci/build_naive.sh | |
| router: | |
| runs-on: [self-hosted, linux, light] | |
| needs: gh_build_image | |
| container: | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| outputs: | |
| affected: ${{ steps.get_affected.outputs.affected }} | |
| steps: | |
| - name: 'Cleanup build folder' | |
| run: | | |
| rm -rf ./* || true | |
| rm -rf ./.??* || true | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - run: git config --global --add safe.directory . | |
| - run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} | |
| - run: git fetch origin main | |
| - name: Get packages affected by changes in the current commit | |
| id: get_affected | |
| run: | | |
| TARGET_DATA=$(. /venv/bin/activate && dl-git range-diff-paths --only-added-commits --base ${{ github.event.pull_request.base.sha }} --head ${{ github.event.pull_request.head.sha }}) | |
| echo "$TARGET_DATA" > /tmp/targets.json | |
| cat /tmp/targets.json | |
| . /venv/bin/activate && detect-affected-packages \ | |
| --repo=/src \ | |
| --changes_file="/tmp/targets.json" \ | |
| --root_pkgs="lib,app,terrarium" \ | |
| --fallback_pkg="terrarium/bi_ci" | tee -a >> "$GITHUB_OUTPUT" | |
| env: | |
| TEST_TARGET_OVERRIDE: ${{ github.event.inputs.test_targets }} | |
| pytest_split: | |
| runs-on: [ self-hosted, linux ] | |
| needs: router | |
| # if: ${{ github.event.inputs.run_mypy_only != 'true' }} | |
| container: | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| outputs: | |
| split_base: ${{ steps.get_split.outputs.split_base }} | |
| split_fat: ${{ steps.get_split.outputs.split_fat }} | |
| steps: | |
| - name: 'Cleanup build folder' | |
| run: | | |
| rm -rf ./* || true | |
| rm -rf ./.??* || true | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| - run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} | |
| - name: Run python script to split job for general and fat runners | |
| id: get_split | |
| run: | | |
| . /venv/bin/activate && echo '${{ needs.router.outputs.affected }}' >> /tmp/dl_test_targets.json | |
| split-pytest-tasks base /src /tmp/dl_test_targets.json | tee -a >> "$GITHUB_OUTPUT" | |
| split-pytest-tasks fat /src /tmp/dl_test_targets.json | tee -a >> "$GITHUB_OUTPUT" | |
| run_tests_base: | |
| runs-on: [ self-hosted, linux, light ] | |
| name: "🐍[pytest]${{ matrix.value }}" | |
| needs: pytest_split | |
| if: ${{ needs.pytest_split.outputs.split_base != '[]' }} | |
| permissions: | |
| packages: write | |
| contents: read | |
| container: | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" | |
| options: -v /var/run/docker.sock:/var/run/docker.sock | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| volumes: | |
| - /var/run/docker.sock:/var/run/docker.sock | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| value: ${{fromJson(needs.pytest_split.outputs.split_base)}} | |
| steps: | |
| - run: echo "Going to run tests for ${{ matrix.value }}" | |
| - run: mkdir /report | |
| - run: echo "Running py tests for ${{ matrix.value }}" | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: "ghcr.io" | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - run: echo compose_path="/src/$(echo ${{ matrix.value }} | cut -d ":" -f1)/" >> "$GITHUB_ENV" | |
| - run: cd "${{ env.compose_path }}" && echo compose_prj="$(basename "$PWD")_$(shuf -i 1000000-1000000000 -n 1)" >> "$GITHUB_ENV" | |
| # We need to set custom compose project name to ensure "unique" container names in the host docker env | |
| - name: run bash script with all logic for starting compose and running tests | |
| run: | | |
| bash /src/ci/execute_test_with_docker_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" \ | |
| WE_ARE_IN_CI=1 | |
| - name: Stop compose if provided | |
| # We could not put this into bash script, since job could be cancelled by user request | |
| if: always() # yes! always | |
| run: bash /src/ci/stop_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" | |
| - uses: actions/upload-artifact@v3 | |
| if: "!cancelled()" | |
| with: | |
| name: "pytest_reports_${{ env.compose_prj }}" | |
| path: /report/ | |
| retention-days: 1 | |
| mypy: | |
| runs-on: [ self-hosted, linux, light ] | |
| needs: router | |
| container: | |
| # until https://github.com/github/docs/issues/25520 is resolved, using vars | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| volumes: | |
| - /var/run/docker.sock:/var/run/docker.sock | |
| name: "🐲 mypy" | |
| timeout-minutes: ${{ inputs.pytest_timeout_minutes && fromJSON(inputs.pytest_timeout_minutes ) || 10 }} | |
| steps: | |
| - name: "Cleanup folder" | |
| run: | | |
| rm -rf ./* || true | |
| rm -rf ./.??* || true | |
| - name: Checkout code, just to get access to .github/execute_mypy... | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| - run: echo '${{ needs.router.outputs.affected }}' >> /tmp/dl_test_targets.json | |
| - name: run mypy | |
| run: | | |
| . /venv/bin/activate | |
| execute-mypy-multi /src /tmp/dl_test_targets.json | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| publish-result: | |
| runs-on: [self-hosted, linux ] | |
| needs: [ "run_tests_base" ] # , "run_tests_fat" ] | |
| if: "!cancelled() && ${{ github.event.inputs.run_mypy_only != 'true' }}" | |
| permissions: | |
| contents: read | |
| issues: read | |
| checks: write | |
| pull-requests: write | |
| container: docker:latest | |
| steps: | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| path: ./report/ | |
| - run: ls -lah ./report | |
| - name: Publish Test Results | |
| uses: datalens-tech/publish-unit-test-result-action@55478522536e0c60b0a4ff0c2bb8ab110d7a0f33 | |
| with: | |
| files: | | |
| ./report/**/*.xml | |
| event_name: ${{ github.event.workflow_run.event }} | |
| report_individual_runs: "true" |