main.yml

name: main

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

on:
  pull_request_target:
    types:
      - opened
      - reopened
      - synchronize
      - labeled
      - unlabeled
  workflow_dispatch:
    inputs:
      test_targets:
        default: "__ALL__"
        description: "Comma separated list to run pytest on, e.g.: `lib/dl_api_lib,lib/dl_core`. Use __ALL__ to run all tests"
      run_mypy_only:
        type: boolean
        default: false
        description: "Check to only run mypy"
      mypy_timeout_minutes:
        type: number
        default: 20
        description: "Timeout for pytest JOB in minutes"

jobs:
  drop-ci-approved-label:
    name: Drop CI Allowed label if PR branch changed
    runs-on: ubuntu-22.04

    permissions:
      contents: read
      pull-requests: write

    steps:
      - name: Drop label
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          [[ ${{ github.event_name }} != "pull_request_target" ]] && exit 0
          
          action=${{ github.event.action }}
          [[ $action == "labeled" || $action == "unlabeled" || $action == "opened" ]] && exit 0
          
          gh pr edit \
            --repo ${{ github.event.repository.full_name }} \
            --remove-label "ci-approved" \
            ${{ github.event.pull_request.number }}

  check-ci-allowed:
    name: Check if CI allowed for PR
    runs-on: ubuntu-22.04

    needs: [ drop-ci-approved-label ]

    permissions:
      contents: read
      pull-requests: read

    env:
      GH_TOKEN: ${{ github.token }}
      OWNER: ${{ github.repository_owner }}
      REPO: ${{ github.event.repository.name }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Check user permissions
        id: check_user_permissions
        working-directory: .github/.scripts
        env:
          USER: ${{ github.actor }}
          PERMISSION: "write"
        run: echo "result=$(./gh_user_check_permission.sh)" >> $GITHUB_OUTPUT

      - name: Check PR permissions
        id: check_pr_permissions
        working-directory: .github/.scripts
        env:
          PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
          LABEL: "ci-approved"
        run: echo "result=$(./gh_pull_request_check_label.sh)" >> $GITHUB_OUTPUT

      - name: Combine permissions
        run: |
          user_allowed=${{ steps.check_user_permissions.outputs.result }}
          pr_allowed=${{ steps.check_pr_permissions.outputs.result }}
          
          [[ $user_allowed == "true" || $pr_allowed == "true" ]] && exit 0
          
          echo "ERROR: User have no permissions to run CI and PR is not approved"
          exit 1

      - name: Checkout branch code
        uses: actions/checkout@v4
        if: ${{ github.event_name != 'workflow_dispatch' && github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name }}
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          path: branch_code
          fetch-depth: 0

      - name: Check fork PR branch contains latest base sha
        if: ${{ github.event_name != 'workflow_dispatch' && github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name }}
        working-directory: branch_code
        run: git branch --contains ${{ github.event.pull_request.base.sha }}

  gh_build_image:
    runs-on: [ self-hosted, linux, light ]

    needs: [ check-ci-allowed ]

    permissions:
      packages: write
      contents: read

    container:
      image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/debian_docker:latest"
      options: -v /var/run/docker.sock:/var/run/docker.sock
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}

    env:
      REPO_OWNER: ${{ github.repository_owner }}
      REPO_NAME: ${{ github.event.repository.name }}
      GIT_SHA: "${{ github.sha }}"

    steps:
      - name: Log in to the Container registry
        uses: docker/login-action@v2
        with:
          registry: "ghcr.io"
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Cleanup build folder
        run: |
          rm -rf ./* || true
          rm -rf ./.??* || true
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}
      - name: Fix dubious ownership
        run: .github/.scripts/git_config_save_directory.sh
        env:
          REPOSITORY_NAME: ${{ github.event.repository.name }}

      - run: |
          export ROOT_DIR="$(realpath .)"
          /bin/bash ci/build_naive.sh

  router:
    runs-on: [ self-hosted, linux, light ]
    needs: gh_build_image
    container:
      image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}"
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    outputs:
      affected: ${{ steps.get_affected.outputs.affected }}
      all_affected_with_skips: ${{ steps.get_affected.outputs.all_affected_with_skips }}
    steps:
    - name: Cleanup build folder
      run: |
        rm -rf ./* || true
        rm -rf ./.??* || true
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        ref: ${{ github.event.pull_request.head.sha }}
        repository: ${{ github.event.pull_request.head.repo.full_name }}
        fetch-depth: 0
    - name: Fix dubious ownership
      run: .github/.scripts/git_config_save_directory.sh
      env:
        REPOSITORY_NAME: ${{ github.event.repository.name }}
    - run: git fetch origin main
    - name: Extract base commit
      run: |
        result="${{ github.event.pull_request.base.sha }}"
        if [ -z "$result" ]; then
            result=$(git rev-parse origin/main)
        fi
        echo "result=$result" >> $GITHUB_OUTPUT
      id: extract_base_commit
    - name: Extract head commit
      run: |
        result="${{ github.event.pull_request.head.sha }}"
        if [ -z "$result" ]; then
            result=$(git rev-parse HEAD)
        fi
        echo "result=$result" >> $GITHUB_OUTPUT
      id: extract_head_commit
    - name: Get packages affected by changes in the current commit
      id: get_affected
      run: |
        TARGET_DATA=$(. /venv/bin/activate && dl-git range-diff-paths --only-added-commits --base ${{ steps.extract_base_commit.outputs.result }} --head ${{ steps.extract_head_commit.outputs.result }})
        echo "$TARGET_DATA" > /tmp/targets.json
        cat /tmp/targets.json
        . /venv/bin/activate && detect-affected-packages \
          --repo=/src \
          --changes_file="/tmp/targets.json" \
          --root_pkgs="lib,app" | tee -a  >> "$GITHUB_OUTPUT"
      env:
        TEST_TARGET_OVERRIDE: ${{ github.event.inputs.test_targets }}
    - name: Print affected packages
      run: |
        echo "Affected packages: ${{ steps.get_affected.outputs.affected }}"
        echo "Affected packages, including skipped in CI: ${{ steps.get_affected.outputs.all_affected_with_skips }}"

  check-skip-tests-label:
    name: Check PR skip tests label
    runs-on: ubuntu-22.04

    needs: [ check-ci-allowed ]

    permissions:
      contents: read
      pull-requests: read

    outputs:
      result: ${{ steps.check_skip_label.outputs.result }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Check PR skip label
        id: check_skip_label
        working-directory: .github/.scripts
        env:
          GH_TOKEN: ${{ github.token }}
          OWNER: ${{ github.repository_owner }}
          REPO: ${{ github.event.repository.name }}
          PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
          LABEL: "tests-force-skipped"
        run: echo "result=$(./gh_pull_request_check_label.sh)" >> $GITHUB_OUTPUT

  pytest_split:
    name: Split pytest into jobs for each test type
    runs-on: [ self-hosted, linux, light ]

    needs: [ router, check-skip-tests-label ]
    if: ${{ needs.check-skip-tests-label.outputs.result != 'true' }}

    # if: ${{ github.event.inputs.run_mypy_only != 'true' }}
    container:
      image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}"
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    outputs:
      split_base: ${{ steps.get_split.outputs.split_base }}
      split_fat: ${{ steps.get_split.outputs.split_fat }}
      split_ext_public: ${{ steps.get_split.outputs.split_ext_public }}
    steps:
      - name: Cleanup build folder
        run: |
          rm -rf ./* || true
          rm -rf ./.??* || true
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          fetch-depth: 1
      - name: Fix dubious ownership
        run: .github/.scripts/git_config_save_directory.sh
        env:
          REPOSITORY_NAME: ${{ github.event.repository.name }}
      - name: Run python script to split job for general and fat runners
        id: get_split
        run: |
          . /venv/bin/activate && echo '${{ needs.router.outputs.affected }}' >> /tmp/dl_test_targets.json
          split-pytest-tasks base,fat,ext_public /src /tmp/dl_test_targets.json true true >> "$GITHUB_OUTPUT"

  run_tests_base:
    runs-on: [ self-hosted, linux, light ]
    name: "🐍[pytest]${{ matrix.value }}"
    timeout-minutes: 60
    needs: pytest_split
    if: ${{ needs.pytest_split.outputs.split_base != '[]' }}
    permissions:
      packages: write
      contents: read
    container:
      image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}"
      options: -v /var/run/docker.sock:/var/run/docker.sock
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
      volumes:
        - /var/run/docker.sock:/var/run/docker.sock
    strategy:
      fail-fast: false
      matrix:
        value: ${{fromJson(needs.pytest_split.outputs.split_base)}}
    steps:
      - run: echo "Going to run tests for  ${{ matrix.value }}"
      - run: mkdir /report
      - run: echo "Running py tests for ${{ matrix.value }}"
      - name: Log in to the Container registry
        uses: docker/login-action@v2
        with:
          registry: "ghcr.io"
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - run: echo compose_path="/src/$(echo ${{ matrix.value }} | cut -d ":" -f1)/" >>  "$GITHUB_ENV"
      - run: cd "${{ env.compose_path }}" && echo compose_prj="$(basename "$PWD")_$(shuf -i 1000000-1000000000 -n 1)" >> "$GITHUB_ENV"
      # We need to set custom compose project name to ensure "unique" container names in the host docker env
      - name: run bash script with all logic for starting compose and running tests
        run: |
          bash /src/ci/execute_test_with_docker_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" \
            WE_ARE_IN_CI=1
      - name: Stop compose if provided
        # We could not put this into bash script, since job could be cancelled by user request
        if: always()  # yes! always
        run: bash /src/ci/stop_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}"
      - uses: actions/upload-artifact@v3
        if: "!cancelled()"
        with:
          name: "pytest_reports_${{ env.compose_prj }}"
          path: /report/
          retention-days: 1

  run_tests_fat:
    runs-on: [ self-hosted, linux, fat ]
    name: "🐍[pytest][fat]${{ matrix.value }}"
    timeout-minutes: 60
    needs: pytest_split
    if: ${{ needs.pytest_split.outputs.split_fat != '[]' }}
    permissions:
      packages: write
      contents: read
    container:
      image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}"
      options: -v /var/run/docker.sock:/var/run/docker.sock
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
      volumes:
        - /var/run/docker.sock:/var/run/docker.sock
    strategy:
      fail-fast: false
      matrix:
        value: ${{fromJson(needs.pytest_split.outputs.split_fat)}}
    steps:
      - run: echo "Going to run tests for  ${{ matrix.value }}"
      - run: mkdir /report
      - run: echo "Running py tests for ${{ matrix.value }}"
      - name: Log in to the Container registry
        uses: docker/login-action@v2
        with:
          registry: "ghcr.io"
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - run: echo compose_path="/src/$(echo ${{ matrix.value }} | cut -d ":" -f1)/" >>  "$GITHUB_ENV"
      - run: cd "${{ env.compose_path }}" && echo compose_prj="$(basename "$PWD")_$(shuf -i 1000000-1000000000 -n 1)" >> "$GITHUB_ENV"
      # We need to set custom compose project name to ensure "unique" container names in the host docker env
      - name: run bash script with all logic for starting compose and running tests
        run: |
          bash /src/ci/execute_test_with_docker_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" \
            WE_ARE_IN_CI=1
      - name: Stop compose if provided
        # We could not put this into bash script, since job could be cancelled by user request
        if: always()  # yes! always
        run: bash /src/ci/stop_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}"
      - uses: actions/upload-artifact@v3
        if: "!cancelled()"
        with:
          name: "pytest_reports_${{ env.compose_prj }}"
          path: /report/
          retention-days: 1

  run_tests_ext_public:
    # using light runners for now, TBD: change when we have ext_public
    runs-on: [ self-hosted, linux, light ]
    name: "🐍[pytest][ext_public]${{ matrix.value }}"
    timeout-minutes: 60
    needs: pytest_split
    if: ${{ needs.pytest_split.outputs.split_ext_public != '[]' }}
    permissions:
      packages: write
      contents: read
    container:
      image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}"
      options: -v /var/run/docker.sock:/var/run/docker.sock
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
      volumes:
        - /var/run/docker.sock:/var/run/docker.sock
    strategy:
      fail-fast: false
      matrix:
        value: ${{fromJson(needs.pytest_split.outputs.split_ext_public)}}
    steps:
      - run: echo "Going to run tests for  ${{ matrix.value }}"
      - run: mkdir /report
      - run: echo "Running py tests for ${{ matrix.value }}"
      - name: Log in to the Container registry
        uses: docker/login-action@v2
        with:
          registry: "ghcr.io"
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - run: echo compose_path="/src/$(echo ${{ matrix.value }} | cut -d ":" -f1)/" >>  "$GITHUB_ENV"
      - run: cd "${{ env.compose_path }}" && echo compose_prj="$(basename "$PWD")_$(shuf -i 1000000-1000000000 -n 1)" >> "$GITHUB_ENV"
      # We need to set custom compose project name to ensure "unique" container names in the host docker env
      - name: run bash script with all logic for starting compose and running tests
        run: |
          bash /src/ci/execute_test_with_docker_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" \
            WE_ARE_IN_CI=1
        env:
          GOOGLE_API_KEY: "${{ secrets.EXT_GOOGLE_API_KEY }}"
          BITRIX_TOKEN: "${{ secrets.EXT_BITRIX_TOKEN }}"
          BITRIX_DATALENS_TOKEN: "${{ secrets.EXT_BITRIX_DATALENS_TOKEN }}"
          METRIKA_OAUTH: "${{ secrets.EXT_METRIKA_OAUTH }}"
          YA_DOCS_API_KEY: "${{ secrets.EXT_YA_DOCS_API_KEY }}"
      - name: Stop compose if provided
        # We could not put this into bash script, since job could be cancelled by user request
        if: always()  # yes! always
        run: bash /src/ci/stop_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}"
      - uses: actions/upload-artifact@v3
        if: "!cancelled()"
        with:
          name: "pytest_reports_${{ env.compose_prj }}"
          path: /report/
          retention-days: 1

  mypy:
    runs-on: [ self-hosted, linux, light ]
    needs: gh_build_image
    container:
      # until https://github.com/github/docs/issues/25520 is resolved, using vars
      image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}"
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
      volumes:
        - /var/run/docker.sock:/var/run/docker.sock
    name: "🐲 mypy"
    timeout-minutes: ${{ inputs.mypy_timeout_minutes && fromJSON(inputs.mypy_timeout_minutes ) || 20 }}
    steps:
      - name: Cleanup build folder
        run: |
          rm -rf ./* || true
          rm -rf ./.??* || true
      - name: Checkout code, just to get access to .github/execute_mypy...
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          fetch-depth: 1
      - name: run mypy
        run: |
          . /venv/bin/activate
          execute-mypy-multi /src --processes=5
        env:
          PYTHONUNBUFFERED: "1"

  test-results:
    name: Test results
    runs-on: ubuntu-22.04

    needs: [ check-skip-tests-label, pytest_split, run_tests_base, run_tests_fat, run_tests_ext_public ]
    if: ${{ always() }}

    steps:
      - name: Test results
        run: |
          skip_tests=${{ needs.check-skip-tests-label.outputs.result }}
          [[ $skip_tests == "true" ]] && exit 0
          
          pytest_split=${{ needs.pytest_split.result }}
          [[ $pytest_split != "success" && $pytest_split != "skipped" ]] && echo "pytest_split failed" && exit 1
          result_base=${{ needs.run_tests_base.result }}
          [[ $result_base != "success" && $result_base != "skipped" ]] && echo "base failed" && exit 1
          result_fat=${{ needs.run_tests_fat.result }}
          [[ $result_fat != "success" && $result_fat != "skipped" ]] && echo "fat failed" && exit 1
          result_ext_public=${{ needs.run_tests_ext_public.result }}
          [[ $result_ext_public != "success" && $result_ext_public != "skipped" ]] && echo "ext_public failed" && exit 1
          
          echo "All tests passed"

  publish-result:
    runs-on: [ self-hosted, linux, light ]
    needs: [ "run_tests_base", "run_tests_fat", "run_tests_ext_public" ]
    if: ${{ !cancelled() && github.event.inputs.run_mypy_only != 'true' && ( needs.run_tests_base.result != 'skipped' || needs.run_tests_fat.result != 'skipped' || needs.run_tests_ext_public.result != 'skipped' )}}
    permissions:
      contents: read
      issues: read
      checks: write
      pull-requests: write
    container: docker:latest
    steps:
      - uses: actions/download-artifact@v3
        with:
          path: ./report/
      - run: ls -lah ./report
      - name: Publish Test Results
        uses: datalens-tech/publish-unit-test-result-action@55478522536e0c60b0a4ff0c2bb8ab110d7a0f33
        with:
          files: |
            ./report/**/*.xml
          event_name: ${{ github.event.workflow_run.event }}
          report_individual_runs: "true"

  codestyle_all:
    runs-on: [ self-hosted, linux, light ]
    needs: gh_build_image
    container:
      # until https://github.com/github/docs/issues/25520 is resolved, using vars
      image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}"
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Cleanup build folder
        run: |
          rm -rf ./* || true
          rm -rf ./.??* || true
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          fetch-depth: 1
      - run: |
          task cq:check_dir -- .
          task cq:check_dir_strict -- .
        env:
          VENV_PATH: /venv