ci: BI-0 fix dependencies #2897
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: main | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| pull_request_target: | |
| types: | |
| - opened | |
| - reopened | |
| - synchronize | |
| - labeled | |
| - unlabeled | |
| workflow_dispatch: | |
| inputs: | |
| test_targets: | |
| default: "__ALL__" | |
| description: "Comma separated list to run pytest on, e.g.: `lib/dl_api_lib,lib/dl_core`. Use __ALL__ to run all tests" | |
| run_mypy_only: | |
| type: boolean | |
| default: false | |
| description: "Check to only run mypy" | |
| mypy_timeout_minutes: | |
| type: number | |
| default: 20 | |
| description: "Timeout for pytest JOB in minutes" | |
| jobs: | |
| drop-ci-approved-label: | |
| name: Drop CI Allowed label if PR branch changed | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| steps: | |
| - name: Drop label | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| [[ ${{ github.event_name }} != "pull_request_target" ]] && exit 0 | |
| action=${{ github.event.action }} | |
| [[ $action == "labeled" || $action == "unlabeled" || $action == "opened" ]] && exit 0 | |
| gh pr edit \ | |
| --repo ${{ github.event.repository.full_name }} \ | |
| --remove-label "ci-approved" \ | |
| ${{ github.event.pull_request.number }} | |
| check-ci-allowed: | |
| name: Check if CI allowed for PR | |
| runs-on: ubuntu-22.04 | |
| needs: [ drop-ci-approved-label ] | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| OWNER: ${{ github.repository_owner }} | |
| REPO: ${{ github.event.repository.name }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Check user permissions | |
| id: check_user_permissions | |
| working-directory: .github/.scripts | |
| env: | |
| USER: ${{ github.actor }} | |
| PERMISSION: "write" | |
| run: echo "result=$(./gh_user_check_permission.sh)" >> $GITHUB_OUTPUT | |
| - name: Check PR permissions | |
| id: check_pr_permissions | |
| working-directory: .github/.scripts | |
| env: | |
| PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} | |
| LABEL: "ci-approved" | |
| run: echo "result=$(./gh_pull_request_check_label.sh)" >> $GITHUB_OUTPUT | |
| - name: Combine permissions | |
| run: | | |
| user_allowed=${{ steps.check_user_permissions.outputs.result }} | |
| pr_allowed=${{ steps.check_pr_permissions.outputs.result }} | |
| [[ $user_allowed == "true" || $pr_allowed == "true" ]] && exit 0 | |
| echo "ERROR: User have no permissions to run CI and PR is not approved" | |
| exit 1 | |
| - name: Checkout branch code | |
| uses: actions/checkout@v4 | |
| if: ${{ github.event_name != 'workflow_dispatch' && github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name }} | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| path: branch_code | |
| fetch-depth: 0 | |
| - name: Check fork PR branch contains latest base sha | |
| if: ${{ github.event_name != 'workflow_dispatch' && github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name }} | |
| working-directory: branch_code | |
| run: git branch --contains ${{ github.event.pull_request.base.sha }} | |
| gh_build_image: | |
| runs-on: [ self-hosted, linux, k8s-runner-build ] | |
| needs: [ check-ci-allowed ] | |
| permissions: | |
| packages: write | |
| contents: read | |
| container: | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/debian_docker:latest" | |
| options: -v /var/run/docker.sock:/var/run/docker.sock | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| env: | |
| REPO_OWNER: ${{ github.repository_owner }} | |
| REPO_NAME: ${{ github.event.repository.name }} | |
| GIT_SHA: "${{ github.sha }}" | |
| steps: | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: "ghcr.io" | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.ROBOT_GITHUB_PACKAGES_WRITE_TOKEN }} | |
| - uses: datalens-tech/cleanup-folder-action@v1 | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| - uses: datalens-tech/fix-dubious-ownership-action@v1 | |
| - run: | | |
| export ROOT_DIR="$(realpath .)" | |
| /bin/bash ci/build_naive.sh | |
| router: | |
| runs-on: [ self-hosted, linux, k8s-runner-no-compose ] | |
| needs: gh_build_image | |
| container: | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| outputs: | |
| affected: ${{ steps.get_affected.outputs.affected }} | |
| all_affected_with_skips: ${{ steps.get_affected.outputs.all_affected_with_skips }} | |
| steps: | |
| - uses: datalens-tech/cleanup-folder-action@v1 | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| fetch-depth: 0 | |
| - uses: datalens-tech/fix-dubious-ownership-action@v1 | |
| - run: git fetch origin main | |
| - name: Extract base commit | |
| run: | | |
| result="${{ github.event.pull_request.base.sha }}" | |
| if [ -z "$result" ]; then | |
| result=$(git rev-parse origin/main) | |
| fi | |
| echo "result=$result" >> $GITHUB_OUTPUT | |
| id: extract_base_commit | |
| - name: Extract head commit | |
| run: | | |
| result="${{ github.event.pull_request.head.sha }}" | |
| if [ -z "$result" ]; then | |
| result=$(git rev-parse HEAD) | |
| fi | |
| echo "result=$result" >> $GITHUB_OUTPUT | |
| id: extract_head_commit | |
| - name: Get packages affected by changes in the current commit | |
| id: get_affected | |
| run: | | |
| TARGET_DATA=$(. /venv/bin/activate && dl-git range-diff-paths --only-added-commits --base ${{ steps.extract_base_commit.outputs.result }} --head ${{ steps.extract_head_commit.outputs.result }}) | |
| echo "$TARGET_DATA" > /tmp/targets.json | |
| cat /tmp/targets.json | |
| . /venv/bin/activate && detect-affected-packages \ | |
| --repo=/src \ | |
| --changes_file="/tmp/targets.json" \ | |
| --root_pkgs="lib,app" | tee -a >> "$GITHUB_OUTPUT" | |
| env: | |
| TEST_TARGET_OVERRIDE: ${{ github.event.inputs.test_targets }} | |
| - name: Print affected packages | |
| run: | | |
| echo "Affected packages: ${{ steps.get_affected.outputs.affected }}" | |
| echo "Affected packages, including skipped in CI: ${{ steps.get_affected.outputs.all_affected_with_skips }}" | |
| check-skip-tests-label: | |
| name: Check PR skip tests label | |
| runs-on: ubuntu-22.04 | |
| needs: [ check-ci-allowed ] | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| outputs: | |
| result: ${{ steps.check_skip_label.outputs.result }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Check PR skip label | |
| id: check_skip_label | |
| working-directory: .github/.scripts | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| OWNER: ${{ github.repository_owner }} | |
| REPO: ${{ github.event.repository.name }} | |
| PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} | |
| LABEL: "tests-force-skipped" | |
| run: echo "result=$(./gh_pull_request_check_label.sh)" >> $GITHUB_OUTPUT | |
| pytest_split: | |
| name: Split pytest into jobs for each test type | |
| runs-on: [ self-hosted, linux, k8s-runner-no-compose ] | |
| needs: [ router, check-skip-tests-label ] | |
| if: ${{ needs.check-skip-tests-label.outputs.result != 'true' }} | |
| # if: ${{ github.event.inputs.run_mypy_only != 'true' }} | |
| container: | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| outputs: | |
| split_base: ${{ steps.get_split.outputs.split_base }} | |
| split_fat: ${{ steps.get_split.outputs.split_fat }} | |
| split_ext_public: ${{ steps.get_split.outputs.split_ext_public }} | |
| split_no_compose: ${{ steps.get_split.outputs.split_no_compose }} | |
| steps: | |
| - uses: datalens-tech/cleanup-folder-action@v1 | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| fetch-depth: 1 | |
| - uses: datalens-tech/fix-dubious-ownership-action@v1 | |
| - name: Run python script to split job for general and fat runners | |
| id: get_split | |
| run: | | |
| . /venv/bin/activate && echo '${{ needs.router.outputs.affected }}' >> /tmp/dl_test_targets.json | |
| split-pytest-tasks base,fat,ext_public,no_compose /src /tmp/dl_test_targets.json true true >> "$GITHUB_OUTPUT" | |
| run_tests_base: | |
| name: "🐍[pytest]${{ matrix.value }}" | |
| runs-on: [ self-hosted, linux, k8s-runner-light ] | |
| permissions: | |
| packages: write | |
| contents: read | |
| needs: pytest_split | |
| if: ${{ needs.pytest_split.outputs.split_base != '[]' }} | |
| container: | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" | |
| options: -v /var/run/docker.sock:/var/run/docker.sock | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| volumes: | |
| - /var/run/docker.sock:/var/run/docker.sock | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| value: ${{fromJson(needs.pytest_split.outputs.split_base)}} | |
| steps: | |
| - run: echo "Going to run tests for ${{ matrix.value }}" | |
| - run: echo "Running py tests for ${{ matrix.value }}" | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: "ghcr.io" | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - run: echo compose_path="/src/$(echo ${{ matrix.value }} | cut -d ":" -f1)/" >> "$GITHUB_ENV" | |
| - run: cd "${{ env.compose_path }}" && echo compose_prj="$(basename "$PWD")_$(shuf -i 1000000-1000000000 -n 1)" >> "$GITHUB_ENV" | |
| # We need to set custom compose project name to ensure "unique" container names in the host docker env | |
| - name: run bash script with all logic for starting compose and running tests | |
| run: | | |
| bash /src/ci/execute_test_with_docker_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" \ | |
| WE_ARE_IN_CI=1 | |
| - name: Stop compose if provided | |
| # We could not put this into bash script, since job could be cancelled by user request | |
| if: always() # yes! always | |
| run: bash /src/ci/stop_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" | |
| run_tests_no_compose: | |
| name: "🐍[pytest][no_compose]${{ matrix.value }}" | |
| runs-on: [ self-hosted, linux, k8s-runner-no-compose ] | |
| permissions: | |
| packages: write | |
| contents: read | |
| needs: pytest_split | |
| if: ${{ needs.pytest_split.outputs.split_no_compose != '[]' }} | |
| container: | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" | |
| options: -v /var/run/docker.sock:/var/run/docker.sock | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| volumes: | |
| - /var/run/docker.sock:/var/run/docker.sock | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| value: ${{fromJson(needs.pytest_split.outputs.split_no_compose)}} | |
| steps: | |
| - run: echo "Going to run tests for ${{ matrix.value }}" | |
| - run: echo "Running py tests for ${{ matrix.value }}" | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: "ghcr.io" | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - run: echo compose_path="/src/$(echo ${{ matrix.value }} | cut -d ":" -f1)/" >> "$GITHUB_ENV" | |
| - run: cd "${{ env.compose_path }}" && echo compose_prj="$(basename "$PWD")_$(shuf -i 1000000-1000000000 -n 1)" >> "$GITHUB_ENV" | |
| # We need to set custom compose project name to ensure "unique" container names in the host docker env | |
| - name: run bash script with all logic for starting compose and running tests | |
| run: | | |
| bash /src/ci/execute_test_with_docker_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" \ | |
| WE_ARE_IN_CI=1 | |
| - name: Stop compose if provided | |
| # We could not put this into bash script, since job could be cancelled by user request | |
| if: always() # yes! always | |
| run: bash /src/ci/stop_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" | |
| run_tests_fat: | |
| name: "🐍[pytest][fat]${{ matrix.value }}" | |
| runs-on: [ self-hosted, linux, k8s-runner-fat ] | |
| permissions: | |
| packages: write | |
| contents: read | |
| needs: pytest_split | |
| if: ${{ needs.pytest_split.outputs.split_fat != '[]' }} | |
| container: | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" | |
| options: -v /var/run/docker.sock:/var/run/docker.sock | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| volumes: | |
| - /var/run/docker.sock:/var/run/docker.sock | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| value: ${{fromJson(needs.pytest_split.outputs.split_fat)}} | |
| steps: | |
| - run: echo "Going to run tests for ${{ matrix.value }}" | |
| - run: echo "Running py tests for ${{ matrix.value }}" | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: "ghcr.io" | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - run: echo compose_path="/src/$(echo ${{ matrix.value }} | cut -d ":" -f1)/" >> "$GITHUB_ENV" | |
| - run: cd "${{ env.compose_path }}" && echo compose_prj="$(basename "$PWD")_$(shuf -i 1000000-1000000000 -n 1)" >> "$GITHUB_ENV" | |
| # We need to set custom compose project name to ensure "unique" container names in the host docker env | |
| - name: run bash script with all logic for starting compose and running tests | |
| run: | | |
| bash /src/ci/execute_test_with_docker_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" \ | |
| WE_ARE_IN_CI=1 | |
| - name: Stop compose if provided | |
| # We could not put this into bash script, since job could be cancelled by user request | |
| if: always() # yes! always | |
| run: bash /src/ci/stop_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" | |
| run_tests_ext_public: | |
| name: "🐍[pytest][ext_public]${{ matrix.value }}" | |
| # using light runners for now, TBD: change when we have ext_public | |
| runs-on: [ self-hosted, linux, k8s-runner-ext-public ] | |
| permissions: | |
| packages: read | |
| contents: read | |
| needs: pytest_split | |
| if: ${{ needs.pytest_split.outputs.split_ext_public != '[]' }} | |
| container: | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" | |
| options: -v /var/run/docker.sock:/var/run/docker.sock | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| volumes: | |
| - /var/run/docker.sock:/var/run/docker.sock | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| value: ${{fromJson(needs.pytest_split.outputs.split_ext_public)}} | |
| steps: | |
| - run: echo "Going to run tests for ${{ matrix.value }}" | |
| - run: echo "Running py tests for ${{ matrix.value }}" | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: "ghcr.io" | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - run: echo compose_path="/src/$(echo ${{ matrix.value }} | cut -d ":" -f1)/" >> "$GITHUB_ENV" | |
| - run: cd "${{ env.compose_path }}" && echo compose_prj="$(basename "$PWD")_$(shuf -i 1000000-1000000000 -n 1)" >> "$GITHUB_ENV" | |
| # We need to set custom compose project name to ensure "unique" container names in the host docker env | |
| - name: run bash script with all logic for starting compose and running tests | |
| run: | | |
| bash /src/ci/execute_test_with_docker_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" \ | |
| WE_ARE_IN_CI=1 | |
| env: | |
| GOOGLE_API_KEY: "${{ secrets.EXT_GOOGLE_API_KEY }}" | |
| BITRIX_DATALENS_TOKEN: "${{ secrets.EXT_BITRIX_DATALENS_TOKEN }}" | |
| METRIKA_OAUTH: "${{ secrets.EXT_METRIKA_OAUTH }}" | |
| YA_DOCS_API_KEY: "${{ secrets.EXT_YA_DOCS_API_KEY }}" | |
| - name: Stop compose if provided | |
| # We could not put this into bash script, since job could be cancelled by user request | |
| if: always() # yes! always | |
| run: bash /src/ci/stop_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" | |
| mypy: | |
| runs-on: [ self-hosted, linux, k8s-runner-no-compose ] | |
| needs: gh_build_image | |
| container: | |
| # until https://github.com/github/docs/issues/25520 is resolved, using vars | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| volumes: | |
| - /var/run/docker.sock:/var/run/docker.sock | |
| name: "🐲 mypy" | |
| timeout-minutes: ${{ inputs.mypy_timeout_minutes && fromJSON(inputs.mypy_timeout_minutes ) || 20 }} | |
| steps: | |
| - uses: datalens-tech/cleanup-folder-action@v1 | |
| - name: Checkout code, just to get access to .github/execute_mypy... | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| fetch-depth: 1 | |
| - name: run mypy | |
| run: | | |
| . /venv/bin/activate | |
| execute-mypy-multi /src --processes=5 | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| test-results: | |
| name: Test results | |
| runs-on: ubuntu-22.04 | |
| needs: | |
| - check-skip-tests-label | |
| - pytest_split | |
| - run_tests_base | |
| - run_tests_fat | |
| - run_tests_no_compose | |
| - run_tests_ext_public | |
| if: always() && !cancelled() | |
| steps: | |
| - name: Collect Result | |
| uses: ovsds/collect-needs-result-action@v1 | |
| with: | |
| needs_json: ${{ toJson(needs) }} | |
| codestyle_all: | |
| runs-on: [ self-hosted, linux, k8s-runner-no-compose ] | |
| needs: gh_build_image | |
| container: | |
| # until https://github.com/github/docs/issues/25520 is resolved, using vars | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - uses: datalens-tech/cleanup-folder-action@v1 | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| fetch-depth: 1 | |
| - run: | | |
| task cq:check_dir -- . | |
| task cq:check_dir_strict -- . | |
| env: | |
| VENV_PATH: /venv | |
| - name: Find directories and check requirements | |
| run: | | |
| task cq:find_and_check_requirements | |
| env: | |
| VENV_PATH: /venv |