diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bfef7168c..0b569aa30 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -102,6 +102,7 @@ jobs: outputs: split_base: ${{ steps.get_split.outputs.split_base }} split_fat: ${{ steps.get_split.outputs.split_fat }} + split_ext_public: ${{ steps.get_split.outputs.split_ext_public }} steps: - name: 'Cleanup build folder' run: | @@ -118,6 +119,7 @@ jobs: . /venv/bin/activate && echo '${{ needs.router.outputs.affected }}' >> /tmp/dl_test_targets.json split-pytest-tasks base /src /tmp/dl_test_targets.json | tee -a >> "$GITHUB_OUTPUT" split-pytest-tasks fat /src /tmp/dl_test_targets.json | tee -a >> "$GITHUB_OUTPUT" + split-pytest-tasks ext_public /src /tmp/dl_test_targets.json | tee -a >> "$GITHUB_OUTPUT" run_tests_base: runs-on: [ self-hosted, linux, light ] @@ -139,6 +141,103 @@ jobs: fail-fast: false matrix: value: ${{fromJson(needs.pytest_split.outputs.split_base)}} + steps: + - run: echo "Going to run tests for ${{ matrix.value }}" + - run: mkdir /report + - run: echo "Running py tests for ${{ matrix.value }}" + - name: Log in to the Container registry + uses: docker/login-action@v2 + with: + registry: "ghcr.io" + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - run: echo compose_path="/src/$(echo ${{ matrix.value }} | cut -d ":" -f1)/" >> "$GITHUB_ENV" + - run: cd "${{ env.compose_path }}" && echo compose_prj="$(basename "$PWD")_$(shuf -i 1000000-1000000000 -n 1)" >> "$GITHUB_ENV" + # We need to set custom compose project name to ensure "unique" container names in the host docker env + - name: run bash script with all logic for starting compose and running tests + run: | + bash /src/ci/execute_test_with_docker_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" \ + WE_ARE_IN_CI=1 + - name: Stop compose if provided + # We could not put this into bash script, since job could be cancelled by user request + if: always() # yes! always + run: bash /src/ci/stop_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" + - uses: actions/upload-artifact@v3 + if: "!cancelled()" + with: + name: "pytest_reports_${{ env.compose_prj }}" + path: /report/ + retention-days: 1 + + run_tests_fat: + runs-on: [ self-hosted, linux, fat ] + name: "🐍[pytest][fat]${{ matrix.value }}" + needs: pytest_split + if: ${{ needs.pytest_split.outputs.split_fat != '[]' }} + permissions: + packages: write + contents: read + container: + image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" + options: -v /var/run/docker.sock:/var/run/docker.sock + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + volumes: + - /var/run/docker.sock:/var/run/docker.sock + strategy: + fail-fast: false + matrix: + value: ${{fromJson(needs.pytest_split.outputs.split_fat)}} + steps: + - run: echo "Going to run tests for ${{ matrix.value }}" + - run: mkdir /report + - run: echo "Running py tests for ${{ matrix.value }}" + - name: Log in to the Container registry + uses: docker/login-action@v2 + with: + registry: "ghcr.io" + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - run: echo compose_path="/src/$(echo ${{ matrix.value }} | cut -d ":" -f1)/" >> "$GITHUB_ENV" + - run: cd "${{ env.compose_path }}" && echo compose_prj="$(basename "$PWD")_$(shuf -i 1000000-1000000000 -n 1)" >> "$GITHUB_ENV" + # We need to set custom compose project name to ensure "unique" container names in the host docker env + - name: run bash script with all logic for starting compose and running tests + run: | + bash /src/ci/execute_test_with_docker_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" \ + WE_ARE_IN_CI=1 + - name: Stop compose if provided + # We could not put this into bash script, since job could be cancelled by user request + if: always() # yes! always + run: bash /src/ci/stop_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" + - uses: actions/upload-artifact@v3 + if: "!cancelled()" + with: + name: "pytest_reports_${{ env.compose_prj }}" + path: /report/ + retention-days: 1 + + run_tests_ext_public: + # using light runners for now, TBD: change when we have ext_public + runs-on: [ self-hosted, linux, light ] + name: "🐍[pytest][ext_public]${{ matrix.value }}" + needs: pytest_split + if: ${{ needs.pytest_split.outputs.split_ext_public != '[]' }} + permissions: + packages: write + contents: read + container: + image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" + options: -v /var/run/docker.sock:/var/run/docker.sock + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + volumes: + - /var/run/docker.sock:/var/run/docker.sock + strategy: + fail-fast: false + matrix: + value: ${{fromJson(needs.pytest_split.outputs.split_ext_public)}} steps: - run: echo "Going to run tests for ${{ matrix.value }}" - run: mkdir /report @@ -205,7 +304,7 @@ jobs: publish-result: runs-on: [self-hosted, linux ] - needs: [ "run_tests_base" ] # , "run_tests_fat" ] + needs: [ "run_tests_base", "run_tests_fat", "run_tests_ext_public" ] if: "!cancelled() && ${{ github.event.inputs.run_mypy_only != 'true' }}" permissions: contents: read @@ -250,35 +349,3 @@ jobs: env: VENV_PATH: /venv SKIP_RUFF: true - - codestyle_changed_without_ruff: - runs-on: [ self-hosted, linux, light ] - needs: gh_build_image - container: - # until https://github.com/github/docs/issues/25520 is resolved, using vars - image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - steps: - - name: 'Cleanup build folder' - run: | - rm -rf ./* || true - rm -rf ./.??* || true - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: true - # https://github.com/actions/runner-images/issues/6775 - - run: git config --global --add safe.directory . - - run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} - - run: | - TARGET=$(. /venv/bin/activate && dl-git range-diff-paths --only-added-commits --base ${{ github.event.pull_request.base.sha }} --head ${{ github.event.pull_request.head.sha }}) - echo $TARGET - task cq:check_target -- "$TARGET" - task cq:check_target_strict -- "$TARGET" 1>/dev/null 2>/dev/null - env: - VENV_PATH: /venv - SKIP_RUFF: true - TEST_TARGET_OVERRIDE: ${{ github.event.inputs.test_targets }} diff --git a/lib/dl_connector_bigquery/pyproject.toml b/lib/dl_connector_bigquery/pyproject.toml index 1419ca6ec..83deab77e 100644 --- a/lib/dl_connector_bigquery/pyproject.toml +++ b/lib/dl_connector_bigquery/pyproject.toml @@ -74,7 +74,7 @@ skip_compose = "true" #[datalens.pytest.ext] #root_dir = "dl_connector_bigquery_tests/ext" #skip_compose = "true" -#labels = ["ext"] +#labels = ["ext_public"] [tool.mypy] diff --git a/lib/dl_connector_snowflake/pyproject.toml b/lib/dl_connector_snowflake/pyproject.toml index e170e63b9..992765848 100644 --- a/lib/dl_connector_snowflake/pyproject.toml +++ b/lib/dl_connector_snowflake/pyproject.toml @@ -78,7 +78,7 @@ testpaths = ["dl_connector_snowflake_tests/ext", "dl_connector_snowflake_tests/u #[datalens.pytest.ext] #root_dir = "dl_connector_snowflake_tests/ext" -#labels = ["ext"] +#labels = ["ext_public"] [datalens.pytest.unit] root_dir = "dl_connector_snowflake_tests/" diff --git a/lib/dl_file_uploader_api_lib/pyproject.toml b/lib/dl_file_uploader_api_lib/pyproject.toml index 311ff9c16..c1f04ea53 100644 --- a/lib/dl_file_uploader_api_lib/pyproject.toml +++ b/lib/dl_file_uploader_api_lib/pyproject.toml @@ -58,7 +58,7 @@ labels = ["fat"] [datalens.pytest.ext] root_dir = "dl_file_uploader_api_lib_tests/" target_path = "ext" -labels = ["ext"] +labels = ["ext_public"] [tool.mypy] warn_unused_configs = true diff --git a/lib/dl_file_uploader_lib/pyproject.toml b/lib/dl_file_uploader_lib/pyproject.toml index d6a8fedb8..0c297056d 100644 --- a/lib/dl_file_uploader_lib/pyproject.toml +++ b/lib/dl_file_uploader_lib/pyproject.toml @@ -49,7 +49,7 @@ target_path = "db" [datalens.pytest.ext] root_dir = "dl_file_uploader_lib_tests" target_path = "ext" -labels = ["ext"] +labels = ["ext_public"] [tool.mypy] warn_unused_configs = true diff --git a/lib/dl_file_uploader_worker_lib/pyproject.toml b/lib/dl_file_uploader_worker_lib/pyproject.toml index 124238e4f..681f1bf74 100644 --- a/lib/dl_file_uploader_worker_lib/pyproject.toml +++ b/lib/dl_file_uploader_worker_lib/pyproject.toml @@ -57,7 +57,7 @@ target_path = "db unit" [datalens.pytest.ext] root_dir = "dl_file_uploader_worker_lib_tests/" target_path = "ext" -labels = ["ext"] +labels = ["ext_public"] [tool.mypy] warn_unused_configs = true