fix(formula): DLFR-1866 CH switch to basic type conv funcs in arrays instead of orNull variants #1556
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: main | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| pull_request_target: | |
| types: | |
| - opened | |
| - reopened | |
| - synchronize | |
| - labeled | |
| - unlabeled | |
| workflow_dispatch: | |
| inputs: | |
| test_targets: | |
| default: "__ALL__" | |
| description: "Comma separated list to run pytest on, e.g.: `lib/dl_api_lib,lib/dl_core`. Use __ALL__ to run all tests" | |
| run_mypy_only: | |
| type: boolean | |
| default: false | |
| description: "Check to only run mypy" | |
| mypy_timeout_minutes: | |
| type: number | |
| default: 20 | |
| description: "Timeout for pytest JOB in minutes" | |
| jobs: | |
| drop-ci-approved-label: | |
| name: Drop CI Allowed label if PR branch changed | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| steps: | |
| - name: Drop label | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| [[ ${{ github.event_name }} != "pull_request_target" ]] && exit 0 | |
| action=${{ github.event.action }} | |
| [[ $action == "labeled" || $action == "unlabeled" || $action == "opened" ]] && exit 0 | |
| gh pr edit \ | |
| --repo ${{ github.event.repository.full_name }} \ | |
| --remove-label "ci-approved" \ | |
| ${{ github.event.pull_request.number }} | |
| check-ci-allowed: | |
| name: Check if CI allowed for PR | |
| runs-on: ubuntu-22.04 | |
| needs: [ drop-ci-approved-label ] | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| OWNER: ${{ github.repository_owner }} | |
| REPO: ${{ github.event.repository.name }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Check user permissions | |
| id: check_user_permissions | |
| working-directory: .github/.scripts | |
| env: | |
| USER: ${{ github.actor }} | |
| PERMISSION: "write" | |
| run: echo "result=$(./gh_user_check_permission.sh)" >> $GITHUB_OUTPUT | |
| - name: Check PR permissions | |
| id: check_pr_permissions | |
| working-directory: .github/.scripts | |
| env: | |
| PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} | |
| LABEL: "ci-approved" | |
| run: echo "result=$(./gh_pull_request_check_label.sh)" >> $GITHUB_OUTPUT | |
| - name: Combine permissions | |
| run: | | |
| user_allowed=${{ steps.check_user_permissions.outputs.result }} | |
| pr_allowed=${{ steps.check_pr_permissions.outputs.result }} | |
| [[ $user_allowed == "true" || $pr_allowed == "true" ]] && exit 0 | |
| echo "ERROR: User have no permissions to run CI and PR is not approved" | |
| exit 1 | |
| - name: Checkout branch code | |
| uses: actions/checkout@v4 | |
| if: ${{ github.event_name != 'workflow_dispatch' && github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name }} | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| path: branch_code | |
| fetch-depth: 0 | |
| - name: Check fork PR branch contains latest base sha | |
| if: ${{ github.event_name != 'workflow_dispatch' && github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name }} | |
| working-directory: branch_code | |
| run: git branch --contains ${{ github.event.pull_request.base.sha }} | |
| gh_build_image: | |
| runs-on: [ self-hosted, linux, light ] | |
| needs: [ check-ci-allowed ] | |
| permissions: | |
| packages: write | |
| contents: read | |
| container: | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/debian_docker:latest" | |
| options: -v /var/run/docker.sock:/var/run/docker.sock | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| env: | |
| REPO_OWNER: ${{ github.repository_owner }} | |
| REPO_NAME: ${{ github.event.repository.name }} | |
| GIT_SHA: "${{ github.sha }}" | |
| steps: | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: "ghcr.io" | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cleanup build folder | |
| run: | | |
| rm -rf ./* || true | |
| rm -rf ./.??* || true | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| - name: Fix dubious ownership | |
| run: .github/.scripts/git_config_save_directory.sh | |
| env: | |
| REPOSITORY_NAME: ${{ github.event.repository.name }} | |
| - run: | | |
| export ROOT_DIR="$(realpath .)" | |
| /bin/bash ci/build_naive.sh | |
| router: | |
| runs-on: [ self-hosted, linux, light ] | |
| needs: gh_build_image | |
| container: | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| outputs: | |
| affected: ${{ steps.get_affected.outputs.affected }} | |
| all_affected_with_skips: ${{ steps.get_affected.outputs.all_affected_with_skips }} | |
| steps: | |
| - name: Cleanup build folder | |
| run: | | |
| rm -rf ./* || true | |
| rm -rf ./.??* || true | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| fetch-depth: 0 | |
| - name: Fix dubious ownership | |
| run: .github/.scripts/git_config_save_directory.sh | |
| env: | |
| REPOSITORY_NAME: ${{ github.event.repository.name }} | |
| - run: git fetch origin main | |
| - name: Extract base commit | |
| run: | | |
| result="${{ github.event.pull_request.base.sha }}" | |
| if [ -z "$result" ]; then | |
| result=$(git rev-parse origin/main) | |
| fi | |
| echo "result=$result" >> $GITHUB_OUTPUT | |
| id: extract_base_commit | |
| - name: Extract head commit | |
| run: | | |
| result="${{ github.event.pull_request.head.sha }}" | |
| if [ -z "$result" ]; then | |
| result=$(git rev-parse HEAD) | |
| fi | |
| echo "result=$result" >> $GITHUB_OUTPUT | |
| id: extract_head_commit | |
| - name: Get packages affected by changes in the current commit | |
| id: get_affected | |
| run: | | |
| TARGET_DATA=$(. /venv/bin/activate && dl-git range-diff-paths --only-added-commits --base ${{ steps.extract_base_commit.outputs.result }} --head ${{ steps.extract_head_commit.outputs.result }}) | |
| echo "$TARGET_DATA" > /tmp/targets.json | |
| cat /tmp/targets.json | |
| . /venv/bin/activate && detect-affected-packages \ | |
| --repo=/src \ | |
| --changes_file="/tmp/targets.json" \ | |
| --root_pkgs="lib,app" | tee -a >> "$GITHUB_OUTPUT" | |
| env: | |
| TEST_TARGET_OVERRIDE: ${{ github.event.inputs.test_targets }} | |
| - name: Print affected packages | |
| run: | | |
| echo "Affected packages: ${{ steps.get_affected.outputs.affected }}" | |
| echo "Affected packages, including skipped in CI: ${{ steps.get_affected.outputs.all_affected_with_skips }}" | |
| check-skip-tests-label: | |
| name: Check PR skip tests label | |
| runs-on: ubuntu-22.04 | |
| needs: [ check-ci-allowed ] | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| outputs: | |
| result: ${{ steps.check_skip_label.outputs.result }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Check PR skip label | |
| id: check_skip_label | |
| working-directory: .github/.scripts | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| OWNER: ${{ github.repository_owner }} | |
| REPO: ${{ github.event.repository.name }} | |
| PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} | |
| LABEL: "tests-force-skipped" | |
| run: echo "result=$(./gh_pull_request_check_label.sh)" >> $GITHUB_OUTPUT | |
| pytest_split: | |
| name: Split pytest into jobs for each test type | |
| runs-on: [ self-hosted, linux, light ] | |
| needs: [ router, check-skip-tests-label ] | |
| if: ${{ needs.check-skip-tests-label.outputs.result != 'true' }} | |
| # if: ${{ github.event.inputs.run_mypy_only != 'true' }} | |
| container: | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| outputs: | |
| split_base: ${{ steps.get_split.outputs.split_base }} | |
| split_fat: ${{ steps.get_split.outputs.split_fat }} | |
| split_ext_public: ${{ steps.get_split.outputs.split_ext_public }} | |
| steps: | |
| - name: Cleanup build folder | |
| run: | | |
| rm -rf ./* || true | |
| rm -rf ./.??* || true | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| fetch-depth: 1 | |
| - name: Fix dubious ownership | |
| run: .github/.scripts/git_config_save_directory.sh | |
| env: | |
| REPOSITORY_NAME: ${{ github.event.repository.name }} | |
| - name: Run python script to split job for general and fat runners | |
| id: get_split | |
| run: | | |
| . /venv/bin/activate && echo '${{ needs.router.outputs.affected }}' >> /tmp/dl_test_targets.json | |
| split-pytest-tasks base,fat,ext_public /src /tmp/dl_test_targets.json true true >> "$GITHUB_OUTPUT" | |
| run_tests_base: | |
| runs-on: [ self-hosted, linux, light ] | |
| name: "🐍[pytest]${{ matrix.value }}" | |
| timeout-minutes: 60 | |
| needs: pytest_split | |
| if: ${{ needs.pytest_split.outputs.split_base != '[]' }} | |
| permissions: | |
| packages: write | |
| contents: read | |
| container: | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" | |
| options: -v /var/run/docker.sock:/var/run/docker.sock | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| volumes: | |
| - /var/run/docker.sock:/var/run/docker.sock | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| value: ${{fromJson(needs.pytest_split.outputs.split_base)}} | |
| steps: | |
| - run: echo "Going to run tests for ${{ matrix.value }}" | |
| - run: mkdir /report | |
| - run: echo "Running py tests for ${{ matrix.value }}" | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: "ghcr.io" | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - run: echo compose_path="/src/$(echo ${{ matrix.value }} | cut -d ":" -f1)/" >> "$GITHUB_ENV" | |
| - run: cd "${{ env.compose_path }}" && echo compose_prj="$(basename "$PWD")_$(shuf -i 1000000-1000000000 -n 1)" >> "$GITHUB_ENV" | |
| # We need to set custom compose project name to ensure "unique" container names in the host docker env | |
| - name: run bash script with all logic for starting compose and running tests | |
| run: | | |
| bash /src/ci/execute_test_with_docker_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" \ | |
| WE_ARE_IN_CI=1 | |
| - name: Stop compose if provided | |
| # We could not put this into bash script, since job could be cancelled by user request | |
| if: always() # yes! always | |
| run: bash /src/ci/stop_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" | |
| - uses: actions/upload-artifact@v3 | |
| if: "!cancelled()" | |
| with: | |
| name: "pytest_reports_${{ env.compose_prj }}" | |
| path: /report/ | |
| retention-days: 1 | |
| run_tests_fat: | |
| runs-on: [ self-hosted, linux, fat ] | |
| name: "🐍[pytest][fat]${{ matrix.value }}" | |
| timeout-minutes: 60 | |
| needs: pytest_split | |
| if: ${{ needs.pytest_split.outputs.split_fat != '[]' }} | |
| permissions: | |
| packages: write | |
| contents: read | |
| container: | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" | |
| options: -v /var/run/docker.sock:/var/run/docker.sock | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| volumes: | |
| - /var/run/docker.sock:/var/run/docker.sock | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| value: ${{fromJson(needs.pytest_split.outputs.split_fat)}} | |
| steps: | |
| - run: echo "Going to run tests for ${{ matrix.value }}" | |
| - run: mkdir /report | |
| - run: echo "Running py tests for ${{ matrix.value }}" | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: "ghcr.io" | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - run: echo compose_path="/src/$(echo ${{ matrix.value }} | cut -d ":" -f1)/" >> "$GITHUB_ENV" | |
| - run: cd "${{ env.compose_path }}" && echo compose_prj="$(basename "$PWD")_$(shuf -i 1000000-1000000000 -n 1)" >> "$GITHUB_ENV" | |
| # We need to set custom compose project name to ensure "unique" container names in the host docker env | |
| - name: run bash script with all logic for starting compose and running tests | |
| run: | | |
| bash /src/ci/execute_test_with_docker_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" \ | |
| WE_ARE_IN_CI=1 | |
| - name: Stop compose if provided | |
| # We could not put this into bash script, since job could be cancelled by user request | |
| if: always() # yes! always | |
| run: bash /src/ci/stop_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" | |
| - uses: actions/upload-artifact@v3 | |
| if: "!cancelled()" | |
| with: | |
| name: "pytest_reports_${{ env.compose_prj }}" | |
| path: /report/ | |
| retention-days: 1 | |
| run_tests_ext_public: | |
| # using light runners for now, TBD: change when we have ext_public | |
| runs-on: [ self-hosted, linux, light ] | |
| name: "🐍[pytest][ext_public]${{ matrix.value }}" | |
| timeout-minutes: 60 | |
| needs: pytest_split | |
| if: ${{ needs.pytest_split.outputs.split_ext_public != '[]' }} | |
| permissions: | |
| packages: write | |
| contents: read | |
| container: | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" | |
| options: -v /var/run/docker.sock:/var/run/docker.sock | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| volumes: | |
| - /var/run/docker.sock:/var/run/docker.sock | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| value: ${{fromJson(needs.pytest_split.outputs.split_ext_public)}} | |
| steps: | |
| - run: echo "Going to run tests for ${{ matrix.value }}" | |
| - run: mkdir /report | |
| - run: echo "Running py tests for ${{ matrix.value }}" | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: "ghcr.io" | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - run: echo compose_path="/src/$(echo ${{ matrix.value }} | cut -d ":" -f1)/" >> "$GITHUB_ENV" | |
| - run: cd "${{ env.compose_path }}" && echo compose_prj="$(basename "$PWD")_$(shuf -i 1000000-1000000000 -n 1)" >> "$GITHUB_ENV" | |
| # We need to set custom compose project name to ensure "unique" container names in the host docker env | |
| - name: run bash script with all logic for starting compose and running tests | |
| run: | | |
| bash /src/ci/execute_test_with_docker_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" \ | |
| WE_ARE_IN_CI=1 | |
| env: | |
| GOOGLE_API_KEY: "${{ secrets.EXT_GOOGLE_API_KEY }}" | |
| BITRIX_TOKEN: "${{ secrets.EXT_BITRIX_TOKEN }}" | |
| BITRIX_DATALENS_TOKEN: "${{ secrets.EXT_BITRIX_DATALENS_TOKEN }}" | |
| METRIKA_OAUTH: "${{ secrets.EXT_METRIKA_OAUTH }}" | |
| YA_DOCS_API_KEY: "${{ secrets.EXT_YA_DOCS_API_KEY }}" | |
| - name: Stop compose if provided | |
| # We could not put this into bash script, since job could be cancelled by user request | |
| if: always() # yes! always | |
| run: bash /src/ci/stop_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" | |
| - uses: actions/upload-artifact@v3 | |
| if: "!cancelled()" | |
| with: | |
| name: "pytest_reports_${{ env.compose_prj }}" | |
| path: /report/ | |
| retention-days: 1 | |
| mypy: | |
| runs-on: [ self-hosted, linux, light ] | |
| needs: gh_build_image | |
| container: | |
| # until https://github.com/github/docs/issues/25520 is resolved, using vars | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| volumes: | |
| - /var/run/docker.sock:/var/run/docker.sock | |
| name: "🐲 mypy" | |
| timeout-minutes: ${{ inputs.mypy_timeout_minutes && fromJSON(inputs.mypy_timeout_minutes ) || 20 }} | |
| steps: | |
| - name: Cleanup build folder | |
| run: | | |
| rm -rf ./* || true | |
| rm -rf ./.??* || true | |
| - name: Checkout code, just to get access to .github/execute_mypy... | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| fetch-depth: 1 | |
| - name: run mypy | |
| run: | | |
| . /venv/bin/activate | |
| execute-mypy-multi /src --processes=5 | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| test-results: | |
| name: Test results | |
| runs-on: ubuntu-22.04 | |
| needs: [ check-skip-tests-label, pytest_split, run_tests_base, run_tests_fat, run_tests_ext_public ] | |
| if: ${{ always() }} | |
| steps: | |
| - name: Test results | |
| run: | | |
| skip_tests=${{ needs.check-skip-tests-label.outputs.result }} | |
| [[ $skip_tests == "true" ]] && exit 0 | |
| pytest_split=${{ needs.pytest_split.result }} | |
| [[ $pytest_split != "success" && $pytest_split != "skipped" ]] && echo "pytest_split failed" && exit 1 | |
| result_base=${{ needs.run_tests_base.result }} | |
| [[ $result_base != "success" && $result_base != "skipped" ]] && echo "base failed" && exit 1 | |
| result_fat=${{ needs.run_tests_fat.result }} | |
| [[ $result_fat != "success" && $result_fat != "skipped" ]] && echo "fat failed" && exit 1 | |
| result_ext_public=${{ needs.run_tests_ext_public.result }} | |
| [[ $result_ext_public != "success" && $result_ext_public != "skipped" ]] && echo "ext_public failed" && exit 1 | |
| echo "All tests passed" | |
| publish-result: | |
| runs-on: [ self-hosted, linux, light ] | |
| needs: [ "run_tests_base", "run_tests_fat", "run_tests_ext_public" ] | |
| if: ${{ !cancelled() && github.event.inputs.run_mypy_only != 'true' && ( needs.run_tests_base.result != 'skipped' || needs.run_tests_fat.result != 'skipped' || needs.run_tests_ext_public.result != 'skipped' )}} | |
| permissions: | |
| contents: read | |
| issues: read | |
| checks: write | |
| pull-requests: write | |
| container: docker:latest | |
| steps: | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| path: ./report/ | |
| - run: ls -lah ./report | |
| - name: Publish Test Results | |
| uses: datalens-tech/publish-unit-test-result-action@55478522536e0c60b0a4ff0c2bb8ab110d7a0f33 | |
| with: | |
| files: | | |
| ./report/**/*.xml | |
| event_name: ${{ github.event.workflow_run.event }} | |
| report_individual_runs: "true" | |
| codestyle_all: | |
| runs-on: [ self-hosted, linux, light ] | |
| needs: gh_build_image | |
| container: | |
| # until https://github.com/github/docs/issues/25520 is resolved, using vars | |
| image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Cleanup build folder | |
| run: | | |
| rm -rf ./* || true | |
| rm -rf ./.??* || true | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| fetch-depth: 1 | |
| - run: | | |
| task cq:check_dir -- . | |
| task cq:check_dir_strict -- . | |
| env: | |
| VENV_PATH: /venv |