Skip to content

fix: BI-0 make optional fields nullable in form configs #2592

fix: BI-0 make optional fields nullable in form configs

fix: BI-0 make optional fields nullable in form configs #2592

Workflow file for this run

name: main
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
on:
pull_request_target:
types:
- opened
- reopened
- synchronize
- labeled
- unlabeled
workflow_dispatch:
inputs:
test_targets:
default: "__ALL__"
description: "Comma separated list to run pytest on, e.g.: `lib/dl_api_lib,lib/dl_core`. Use __ALL__ to run all tests"
run_mypy_only:
type: boolean
default: false
description: "Check to only run mypy"
mypy_timeout_minutes:
type: number
default: 20
description: "Timeout for pytest JOB in minutes"
jobs:
drop-ci-approved-label:
name: Drop CI Allowed label if PR branch changed
runs-on: ubuntu-22.04
permissions:
contents: read
pull-requests: write
steps:
- name: Drop label
env:
GH_TOKEN: ${{ github.token }}
run: |
[[ ${{ github.event_name }} != "pull_request_target" ]] && exit 0
action=${{ github.event.action }}
[[ $action == "labeled" || $action == "unlabeled" || $action == "opened" ]] && exit 0
gh pr edit \
--repo ${{ github.event.repository.full_name }} \
--remove-label "ci-approved" \
${{ github.event.pull_request.number }}
check-ci-allowed:
name: Check if CI allowed for PR
runs-on: ubuntu-22.04
needs: [ drop-ci-approved-label ]
permissions:
contents: read
pull-requests: read
env:
GH_TOKEN: ${{ github.token }}
OWNER: ${{ github.repository_owner }}
REPO: ${{ github.event.repository.name }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Check user permissions
id: check_user_permissions
working-directory: .github/.scripts
env:
USER: ${{ github.actor }}
PERMISSION: "write"
run: echo "result=$(./gh_user_check_permission.sh)" >> $GITHUB_OUTPUT
- name: Check PR permissions
id: check_pr_permissions
working-directory: .github/.scripts
env:
PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
LABEL: "ci-approved"
run: echo "result=$(./gh_pull_request_check_label.sh)" >> $GITHUB_OUTPUT
- name: Combine permissions
run: |
user_allowed=${{ steps.check_user_permissions.outputs.result }}
pr_allowed=${{ steps.check_pr_permissions.outputs.result }}
[[ $user_allowed == "true" || $pr_allowed == "true" ]] && exit 0
echo "ERROR: User have no permissions to run CI and PR is not approved"
exit 1
- name: Checkout branch code
uses: actions/checkout@v4
if: ${{ github.event_name != 'workflow_dispatch' && github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name }}
with:
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
path: branch_code
fetch-depth: 0
- name: Check fork PR branch contains latest base sha
if: ${{ github.event_name != 'workflow_dispatch' && github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name }}
working-directory: branch_code
run: git branch --contains ${{ github.event.pull_request.base.sha }}
gh_build_image:
runs-on: [ self-hosted, linux, light ]
needs: [ check-ci-allowed ]
permissions:
packages: write
contents: read
container:
image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/debian_docker:latest"
options: -v /var/run/docker.sock:/var/run/docker.sock
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
env:
REPO_OWNER: ${{ github.repository_owner }}
REPO_NAME: ${{ github.event.repository.name }}
GIT_SHA: "${{ github.sha }}"
steps:
- name: Log in to the Container registry
uses: docker/login-action@v2
with:
registry: "ghcr.io"
username: ${{ github.actor }}
password: ${{ secrets.ROBOT_GITHUB_PACKAGES_WRITE_TOKEN }}
- name: Cleanup build folder
run: |
rm -rf ./* || true
rm -rf ./.??* || true
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- name: Fix dubious ownership
run: .github/.scripts/git_config_save_directory.sh
env:
REPOSITORY_NAME: ${{ github.event.repository.name }}
- run: |
export ROOT_DIR="$(realpath .)"
/bin/bash ci/build_naive.sh
router:
runs-on: [ self-hosted, linux, k8s-runner-no-compose ]
needs: gh_build_image
container:
image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}"
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
outputs:
affected: ${{ steps.get_affected.outputs.affected }}
all_affected_with_skips: ${{ steps.get_affected.outputs.all_affected_with_skips }}
steps:
- name: Cleanup build folder
run: |
rm -rf ./* || true
rm -rf ./.??* || true
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
fetch-depth: 0
- name: Fix dubious ownership
run: .github/.scripts/git_config_save_directory.sh
env:
REPOSITORY_NAME: ${{ github.event.repository.name }}
- run: git fetch origin main
- name: Extract base commit
run: |
result="${{ github.event.pull_request.base.sha }}"
if [ -z "$result" ]; then
result=$(git rev-parse origin/main)
fi
echo "result=$result" >> $GITHUB_OUTPUT
id: extract_base_commit
- name: Extract head commit
run: |
result="${{ github.event.pull_request.head.sha }}"
if [ -z "$result" ]; then
result=$(git rev-parse HEAD)
fi
echo "result=$result" >> $GITHUB_OUTPUT
id: extract_head_commit
- name: Get packages affected by changes in the current commit
id: get_affected
run: |
TARGET_DATA=$(. /venv/bin/activate && dl-git range-diff-paths --only-added-commits --base ${{ steps.extract_base_commit.outputs.result }} --head ${{ steps.extract_head_commit.outputs.result }})
echo "$TARGET_DATA" > /tmp/targets.json
cat /tmp/targets.json
. /venv/bin/activate && detect-affected-packages \
--repo=/src \
--changes_file="/tmp/targets.json" \
--root_pkgs="lib,app" | tee -a >> "$GITHUB_OUTPUT"
env:
TEST_TARGET_OVERRIDE: ${{ github.event.inputs.test_targets }}
- name: Print affected packages
run: |
echo "Affected packages: ${{ steps.get_affected.outputs.affected }}"
echo "Affected packages, including skipped in CI: ${{ steps.get_affected.outputs.all_affected_with_skips }}"
check-skip-tests-label:
name: Check PR skip tests label
runs-on: ubuntu-22.04
needs: [ check-ci-allowed ]
permissions:
contents: read
pull-requests: read
outputs:
result: ${{ steps.check_skip_label.outputs.result }}
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Check PR skip label
id: check_skip_label
working-directory: .github/.scripts
env:
GH_TOKEN: ${{ github.token }}
OWNER: ${{ github.repository_owner }}
REPO: ${{ github.event.repository.name }}
PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
LABEL: "tests-force-skipped"
run: echo "result=$(./gh_pull_request_check_label.sh)" >> $GITHUB_OUTPUT
pytest_split:
name: Split pytest into jobs for each test type
runs-on: [ self-hosted, linux, k8s-runner-no-compose ]
needs: [ router, check-skip-tests-label ]
if: ${{ needs.check-skip-tests-label.outputs.result != 'true' }}
# if: ${{ github.event.inputs.run_mypy_only != 'true' }}
container:
image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}"
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
outputs:
split_base: ${{ steps.get_split.outputs.split_base }}
split_fat: ${{ steps.get_split.outputs.split_fat }}
split_ext_public: ${{ steps.get_split.outputs.split_ext_public }}
split_no_compose: ${{ steps.get_split.outputs.split_no_compose }}
steps:
- name: Cleanup build folder
run: |
rm -rf ./* || true
rm -rf ./.??* || true
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
fetch-depth: 1
- name: Fix dubious ownership
run: .github/.scripts/git_config_save_directory.sh
env:
REPOSITORY_NAME: ${{ github.event.repository.name }}
- name: Run python script to split job for general and fat runners
id: get_split
run: |
. /venv/bin/activate && echo '${{ needs.router.outputs.affected }}' >> /tmp/dl_test_targets.json
split-pytest-tasks base,fat,ext_public,no_compose /src /tmp/dl_test_targets.json true true >> "$GITHUB_OUTPUT"
run_tests_base:
name: "🐍[pytest]${{ matrix.value }}"
runs-on: [ self-hosted, linux, light ]
permissions:
packages: write
contents: read
needs: pytest_split
if: ${{ needs.pytest_split.outputs.split_base != '[]' }}
container:
image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}"
options: -v /var/run/docker.sock:/var/run/docker.sock
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
volumes:
- /var/run/docker.sock:/var/run/docker.sock
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
value: ${{fromJson(needs.pytest_split.outputs.split_base)}}
steps:
- run: echo "Going to run tests for ${{ matrix.value }}"
- run: mkdir /report
- run: echo "Running py tests for ${{ matrix.value }}"
- name: Log in to the Container registry
uses: docker/login-action@v2
with:
registry: "ghcr.io"
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- run: echo compose_path="/src/$(echo ${{ matrix.value }} | cut -d ":" -f1)/" >> "$GITHUB_ENV"
- run: cd "${{ env.compose_path }}" && echo compose_prj="$(basename "$PWD")_$(shuf -i 1000000-1000000000 -n 1)" >> "$GITHUB_ENV"
# We need to set custom compose project name to ensure "unique" container names in the host docker env
- name: run bash script with all logic for starting compose and running tests
run: |
bash /src/ci/execute_test_with_docker_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" \
WE_ARE_IN_CI=1
- name: Stop compose if provided
# We could not put this into bash script, since job could be cancelled by user request
if: always() # yes! always
run: bash /src/ci/stop_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}"
- uses: actions/upload-artifact@v3
if: "!cancelled()"
with:
name: "pytest_reports_${{ env.compose_prj }}"
path: /report/
retention-days: 1
run_tests_no_compose:
name: "🐍[pytest][no_compose]${{ matrix.value }}"
runs-on: [ self-hosted, linux, k8s-runner-no-compose ]
permissions:
packages: write
contents: read
needs: pytest_split
if: ${{ needs.pytest_split.outputs.split_no_compose != '[]' }}
container:
image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}"
options: -v /var/run/docker.sock:/var/run/docker.sock
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
volumes:
- /var/run/docker.sock:/var/run/docker.sock
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
value: ${{fromJson(needs.pytest_split.outputs.split_no_compose)}}
steps:
- run: echo "Going to run tests for ${{ matrix.value }}"
- run: mkdir /report
- run: echo "Running py tests for ${{ matrix.value }}"
- name: Log in to the Container registry
uses: docker/login-action@v2
with:
registry: "ghcr.io"
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- run: echo compose_path="/src/$(echo ${{ matrix.value }} | cut -d ":" -f1)/" >> "$GITHUB_ENV"
- run: cd "${{ env.compose_path }}" && echo compose_prj="$(basename "$PWD")_$(shuf -i 1000000-1000000000 -n 1)" >> "$GITHUB_ENV"
# We need to set custom compose project name to ensure "unique" container names in the host docker env
- name: run bash script with all logic for starting compose and running tests
run: |
bash /src/ci/execute_test_with_docker_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" \
WE_ARE_IN_CI=1
- name: Stop compose if provided
# We could not put this into bash script, since job could be cancelled by user request
if: always() # yes! always
run: bash /src/ci/stop_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}"
- uses: actions/upload-artifact@v3
if: "!cancelled()"
with:
name: "pytest_reports_${{ env.compose_prj }}"
path: /report/
retention-days: 1
run_tests_fat:
name: "🐍[pytest][fat]${{ matrix.value }}"
runs-on: [ self-hosted, linux, fat ]
permissions:
packages: write
contents: read
needs: pytest_split
if: ${{ needs.pytest_split.outputs.split_fat != '[]' }}
container:
image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}"
options: -v /var/run/docker.sock:/var/run/docker.sock
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
volumes:
- /var/run/docker.sock:/var/run/docker.sock
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
value: ${{fromJson(needs.pytest_split.outputs.split_fat)}}
steps:
- run: echo "Going to run tests for ${{ matrix.value }}"
- run: mkdir /report
- run: echo "Running py tests for ${{ matrix.value }}"
- name: Log in to the Container registry
uses: docker/login-action@v2
with:
registry: "ghcr.io"
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- run: echo compose_path="/src/$(echo ${{ matrix.value }} | cut -d ":" -f1)/" >> "$GITHUB_ENV"
- run: cd "${{ env.compose_path }}" && echo compose_prj="$(basename "$PWD")_$(shuf -i 1000000-1000000000 -n 1)" >> "$GITHUB_ENV"
# We need to set custom compose project name to ensure "unique" container names in the host docker env
- name: run bash script with all logic for starting compose and running tests
run: |
bash /src/ci/execute_test_with_docker_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" \
WE_ARE_IN_CI=1
- name: Stop compose if provided
# We could not put this into bash script, since job could be cancelled by user request
if: always() # yes! always
run: bash /src/ci/stop_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}"
- uses: actions/upload-artifact@v3
if: "!cancelled()"
with:
name: "pytest_reports_${{ env.compose_prj }}"
path: /report/
retention-days: 1
run_tests_ext_public:
name: "🐍[pytest][ext_public]${{ matrix.value }}"
# using light runners for now, TBD: change when we have ext_public
runs-on: [ self-hosted, linux, k8s-runner-ext-public ]
permissions:
packages: read
contents: read
needs: pytest_split
if: ${{ needs.pytest_split.outputs.split_ext_public != '[]' }}
container:
image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}"
options: -v /var/run/docker.sock:/var/run/docker.sock
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
volumes:
- /var/run/docker.sock:/var/run/docker.sock
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
value: ${{fromJson(needs.pytest_split.outputs.split_ext_public)}}
steps:
- run: echo "Going to run tests for ${{ matrix.value }}"
- run: mkdir /report
- run: echo "Running py tests for ${{ matrix.value }}"
- name: Log in to the Container registry
uses: docker/login-action@v2
with:
registry: "ghcr.io"
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- run: echo compose_path="/src/$(echo ${{ matrix.value }} | cut -d ":" -f1)/" >> "$GITHUB_ENV"
- run: cd "${{ env.compose_path }}" && echo compose_prj="$(basename "$PWD")_$(shuf -i 1000000-1000000000 -n 1)" >> "$GITHUB_ENV"
# We need to set custom compose project name to ensure "unique" container names in the host docker env
- name: run bash script with all logic for starting compose and running tests
run: |
bash /src/ci/execute_test_with_docker_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" \
WE_ARE_IN_CI=1
env:
GOOGLE_API_KEY: "${{ secrets.EXT_GOOGLE_API_KEY }}"
BITRIX_DATALENS_TOKEN: "${{ secrets.EXT_BITRIX_DATALENS_TOKEN }}"
METRIKA_OAUTH: "${{ secrets.EXT_METRIKA_OAUTH }}"
YA_DOCS_API_KEY: "${{ secrets.EXT_YA_DOCS_API_KEY }}"
- name: Stop compose if provided
# We could not put this into bash script, since job could be cancelled by user request
if: always() # yes! always
run: bash /src/ci/stop_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}"
- uses: actions/upload-artifact@v3
if: "!cancelled()"
with:
name: "pytest_reports_${{ env.compose_prj }}"
path: /report/
retention-days: 1
mypy:
runs-on: [ self-hosted, linux, k8s-runner-no-compose ]
needs: gh_build_image
container:
# until https://github.com/github/docs/issues/25520 is resolved, using vars
image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}"
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
volumes:
- /var/run/docker.sock:/var/run/docker.sock
name: "🐲 mypy"
timeout-minutes: ${{ inputs.mypy_timeout_minutes && fromJSON(inputs.mypy_timeout_minutes ) || 20 }}
steps:
- name: Cleanup build folder
run: |
rm -rf ./* || true
rm -rf ./.??* || true
- name: Checkout code, just to get access to .github/execute_mypy...
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
fetch-depth: 1
- name: run mypy
run: |
. /venv/bin/activate
execute-mypy-multi /src --processes=5
env:
PYTHONUNBUFFERED: "1"
test-results:
name: Test results
runs-on: ubuntu-22.04
needs: [ check-skip-tests-label, pytest_split, run_tests_base, run_tests_fat, run_tests_ext_public ]
if: ${{ always() }}
steps:
- name: Test results
run: |
skip_tests=${{ needs.check-skip-tests-label.outputs.result }}
[[ $skip_tests == "true" ]] && exit 0
pytest_split=${{ needs.pytest_split.result }}
[[ $pytest_split != "success" && $pytest_split != "skipped" ]] && echo "pytest_split failed" && exit 1
result_base=${{ needs.run_tests_base.result }}
[[ $result_base != "success" && $result_base != "skipped" ]] && echo "base failed" && exit 1
result_fat=${{ needs.run_tests_fat.result }}
[[ $result_fat != "success" && $result_fat != "skipped" ]] && echo "fat failed" && exit 1
result_ext_public=${{ needs.run_tests_ext_public.result }}
[[ $result_ext_public != "success" && $result_ext_public != "skipped" ]] && echo "ext_public failed" && exit 1
echo "All tests passed"
codestyle_all:
runs-on: [ self-hosted, linux, k8s-runner-no-compose ]
needs: gh_build_image
container:
# until https://github.com/github/docs/issues/25520 is resolved, using vars
image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}"
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
steps:
- name: Cleanup build folder
run: |
rm -rf ./* || true
rm -rf ./.??* || true
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
fetch-depth: 1
- run: |
task cq:check_dir -- .
task cq:check_dir_strict -- .
env:
VENV_PATH: /venv
- name: Find directories and check requirements
run: |
task cq:find_and_check_requirements
env:
VENV_PATH: /venv