From d51298d121bb5c4779c24d5746aeff1f00198480 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 17 Mar 2024 12:58:33 +0100 Subject: [PATCH] Use common image build workflows in pull-request-target workflow We left a little duplication of the code that has been used to build images in "Build Images" workflow - that is run as "Pull request target" workflow - mainly because of the security concerns and the way how we are replacing the source for actions, workflows and CI scripts from the incoming PRs with the one coming from the target branch. This change moves parts of the code that was used to replace the scripts to the shared workflows and removes the composite actions that we used in the past as common code. As part of that change it turned out that there was a bug in target-commit-sha usage for PRs coming from the forks where rather than using the merge commit, we were using the original commit coming from the fork. This was the reason why often we asked the users to rebase their PRs where some new breaking changes were added in the workflows or new dependencies added. With using merge commit, we should be experiencing the "please rebase to take into account the latest version of CI or Airflow" far less frequently. --- .github/actions/build-ci-images/action.yml | 55 ------ .github/actions/build-prod-images/action.yml | 108 ----------- .github/workflows/build-images.yml | 191 ++++--------------- .github/workflows/ci-image-build.yml | 73 ++++++- .github/workflows/prod-image-build.yml | 87 +++++++-- 5 files changed, 181 insertions(+), 333 deletions(-) delete mode 100644 .github/actions/build-ci-images/action.yml delete mode 100644 .github/actions/build-prod-images/action.yml diff --git a/.github/actions/build-ci-images/action.yml b/.github/actions/build-ci-images/action.yml deleted file mode 100644 index b743aad41d307..0000000000000 --- a/.github/actions/build-ci-images/action.yml +++ /dev/null @@ -1,55 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -name: 'Build CI images' -description: 'Build CI images' -inputs: - python-version: - description: 'Python version to use' - required: true -runs: - using: "composite" - steps: - - name: "Install Breeze" - uses: ./.github/actions/breeze - - name: "Check if dependencies are properly regenerated" - shell: bash - run: | - pip install rich>=12.4.4 pyyaml - python scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py - if: env.UPGRADE_TO_NEWER_DEPENDENCIES != 'false' - - name: Login to ghcr.io - shell: bash - run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - name: "Build & Push AMD64 CI images ${{ env.IMAGE_TAG }} ${{ inputs.python-version }}" - shell: bash - run: > - breeze ci-image build --push --tag-as-latest --upgrade-on-failure - --python "${{ inputs.python-version }}" - - name: "Source constraints: ${{ inputs.python-version }}" - shell: bash - run: > - breeze release-management generate-constraints --python "${{ inputs.python-version }}" - --airflow-constraints-mode constraints-source-providers - - name: "Upload constraint artifacts" - uses: actions/upload-artifact@v4 - with: - name: source-constraints-${{ inputs.python-version }} - path: ./files/constraints-*/constraints-source-providers-*.txt - retention-days: 7 - if-no-files-found: error diff --git a/.github/actions/build-prod-images/action.yml b/.github/actions/build-prod-images/action.yml deleted file mode 100644 index 38c79e11beb48..0000000000000 --- a/.github/actions/build-prod-images/action.yml +++ /dev/null @@ -1,108 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -name: 'Build PROD images' -description: 'Build PROD images' -inputs: - python-version: - description: 'Python versions to use' - required: true - type: string - build-provider-packages: - description: 'Whether to build provider packages from sources (true/false)' - required: true - type: string - chicken-egg-providers: - description: 'List space separated chicken-egg provider packages to build from sources' - required: true - type: string - push-image: - description: 'Whether to push image (true/false)' - required: true - type: string -runs: - using: "composite" - steps: - - name: "Regenerate dependencies in case they was modified manually so that we can build an image" - shell: bash - run: | - pip install rich>=12.4.4 pyyaml - python scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py - if: env.UPGRADE_TO_NEWER_DEPENDENCIES != 'false' - - name: "Cleanup dist and context file" - shell: bash - run: rm -fv ./dist/* ./docker-context-files/* - - name: "Prepare providers packages" - shell: bash - run: > - breeze release-management prepare-provider-packages - --package-list-file ./prod_image_installed_providers.txt - --package-format wheel --version-suffix-for-pypi dev0 - if: ${{ inputs.build-provider-packages == 'true' }} - - name: "Prepare chicken-eggs provider packages" - # In case of provider packages which use latest dev0 version of providers, we should prepare them - # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency - # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0 - shell: bash - run: > - breeze release-management prepare-provider-packages - --package-format wheel --version-suffix-for-pypi dev0 ${{ inputs.chicken-egg-providers }} - if: ${{ inputs.build-provider-packages != 'true' && inputs.chicken-egg-providers != '' }} - - name: "Prepare airflow package" - shell: bash - run: > - breeze release-management prepare-airflow-package - --package-format wheel --version-suffix-for-pypi dev0 - - name: "Copy dist packages to docker-context files" - shell: bash - run: cp -v --no-preserve=mode,ownership ./dist/*.whl ./docker-context-files - - name: "Download constraints from the CI build" - uses: actions/download-artifact@v4 - with: - name: source-constraints-${{ inputs.python-version }} - path: ./docker-context-files - if: ${{ inputs.build-provider-packages == 'true' }} - - name: "Download constraints" - uses: actions/download-artifact@v4 - with: - name: constraints - path: ./docker-context-files - if: ${{ inputs.build-provider-packages != 'true' }} - - name: Login to ghcr.io - shell: bash - run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - name: "Build PROD images w/ source providers ${{ inputs.python-version }}:${{ env.IMAGE_TAG }}" - shell: bash - run: > - breeze prod-image build --tag-as-latest - --install-packages-from-context --airflow-constraints-mode constraints-source-providers - --use-constraints-for-context-packages --python "${{ inputs.python-version }}" - env: - COMMIT_SHA: ${{ github.sha }} - PUSH: ${{ inputs.push-image }} - if: ${{ inputs.build-provider-packages == 'true' }} - - name: "Build PROD images with PyPi providers ${{ inputs.python-version }}:${{ env.IMAGE_TAG }}" - shell: bash - run: > - breeze prod-image build --tag-as-latest - --install-packages-from-context --airflow-constraints-mode constraints - --use-constraints-for-context-packages --python "${{ inputs.python-version }}" - env: - COMMIT_SHA: ${{ github.sha }} - PUSH: ${{ inputs.push-image }} - if: ${{ inputs.build-provider-packages != 'true' }} diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 5070c1fa52fae..2f0232983ad47 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -159,168 +159,59 @@ jobs: GITHUB_CONTEXT: ${{ toJson(github) }} build-ci-images: - strategy: - fail-fast: true - matrix: - python-version: ${{fromJson(needs.build-info.outputs.python-versions)}} + name: Build CI images permissions: contents: read packages: write - timeout-minutes: 80 - name: Build CI image ${{ matrix.python-version }} - runs-on: ["ubuntu-22.04"] + secrets: inherit needs: [build-info] + uses: ./.github/workflows/ci-image-build.yml + # Only run this it if the PR comes from fork, otherwise build will be done "in-PR-workflow" if: | needs.build-info.outputs.ci-image-build == 'true' && github.event.pull_request.head.repo.full_name != 'apache/airflow' - env: - DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} - DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} - RUNS_ON: "${{ needs.build-info.outputs.runs-on }}" - BACKEND: sqlite - VERSION_SUFFIX_FOR_PYPI: "dev0" - USE_UV: "true" - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - uses: actions/checkout@v4 - with: - ref: ${{ needs.build-info.outputs.target-commit-sha }} - persist-credentials: false - #################################################################################################### - # BE VERY CAREFUL HERE! THIS LINE AND THE END OF THE WARNING. HERE WE CHECK OUT THE TARGET - # COMMIT AND ITS PARENT TO BE ABLE TO COMPARE THEM BUT ALSO TO BE ABLE TO BUILD THE IMAGE FROM - # THE INCOMING PR, RATHER THAN FROM TARGET BRANCH. THIS IS A SECURITY RISK, BECAUSE THE PR - # CAN CONTAIN ANY CODE AND WE EXECUTE IT HERE. THEREFORE, WE NEED TO BE VERY CAREFUL WHAT WE - # DO HERE. WE SHOULD NOT EXECUTE ANY CODE THAT COMES FROM THE PR. WE SHOULD NOT RUN ANY BREEZE - # COMMAND NOR SCRIPTS NOR COMPOSITE ACTIONS. WE SHOULD ONLY RUN CODE THAT IS EMBEDDED DIRECTLY IN - # THIS WORKFLOW - BECAUSE THIS IS THE ONLY CODE THAT WE CAN TRUST. - #################################################################################################### - - name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there. - uses: actions/checkout@v4 - with: - path: "target-airflow" - ref: ${{ github.base_ref }} - persist-credentials: false - if: needs.build-info.outputs.is-committer-build != 'true' - - name: > - Replace "scripts/ci", "dev" and ".github/actions" with the target branch - so that the those directories are not coming from the PR - shell: bash - run: | - echo - echo -e "\033[33m Replace scripts, dev, actions with target branch for non-committer builds!\033[0m" - echo - rm -rfv "scripts/ci" - mv -v "target-airflow/scripts/ci" "scripts" - rm -rfv "dev" - mv -v "target-airflow/dev" "." - rm -rfv ".github/actions" - mv -v "target-airflow/.github/actions" ".github" - if: needs.build-info.outputs.is-committer-build != 'true' - #################################################################################################### - # HERE IT'S A BIT SAFER. THE `dev`, `scripts/ci` AND `.github/actions` ARE NOW COMING FROM THE - # BASE_REF - WHICH IS THE TARGET BRANCH OF THE PR. WE CAN TRUST THAT THOSE SCRIPTS ARE SAVE TO RUN. - # ALL THE REST OF THE CODE COMES FROM THE PR, AND FOR EXAMPLE THE CODE IN THE `Dockerfile.ci` CAN - # BE RUN SAFELY AS PART OF DOCKER BUILD. BECAUSE IT RUNS INSIDE THE DOCKER CONTAINER AND IT IS - # ISOLATED FROM THE RUNNER. - #################################################################################################### - - name: Cleanup docker - uses: ./.github/actions/cleanup-docker - - name: Build CI Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}} - uses: ./.github/actions/build-ci-images - with: - python-version: ${{ matrix.python-version }} - env: - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} - PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} - DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} - BUILD_TIMEOUT_MINUTES: 70 + with: + runs-on: ${{ needs.build-info.outputs.runs-on }} + do-build: ${{ needs.build-info.outputs.ci-image-build }} + target-commit-sha: ${{ needs.build-info.outputs.target-commit-sha }} + pull-request-target: "true" + is-committer-build: ${{ needs.build-info.outputs.is-committer-build }} + push-image: "true" + upload-constraints: "true" + use-uv: "true" + image-tag: ${{ needs.build-info.outputs.image-tag }} + python-versions: ${{ needs.build-info.outputs.python-versions }} + branch: ${{ needs.build-info.outputs.default-branch }} + constraints-branch: ${{ needs.build-info.outputs.constraints-branch }} + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + docker-cache: ${{ needs.build-info.outputs.cache-directive }} + build-prod-images: - strategy: - fail-fast: true - matrix: - python-version: ${{fromJson(needs.build-info.outputs.python-versions)}} + name: Build PROD images permissions: contents: read packages: write - timeout-minutes: 80 - name: Build PROD image ${{ matrix.python-version }} - runs-on: ["ubuntu-22.04"] - needs: [build-info, build-ci-images] + secrets: inherit + uses: ./.github/workflows/prod-image-build.yml + # Only run this it if the PR comes from fork, otherwise build will be done "in-PR-workflow" if: | needs.build-info.outputs.prod-image-build == 'true' && github.event.pull_request.head.repo.full_name != 'apache/airflow' - env: - DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} - DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} - RUNS_ON: "${{ needs.build-info.outputs.runs-on }}" - BACKEND: sqlite - VERSION_SUFFIX_FOR_PYPI: "dev0" - INCLUDE_NOT_READY_PROVIDERS: "true" - USE_UV: "true" - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - uses: actions/checkout@v4 - with: - ref: ${{ needs.build-info.outputs.target-commit-sha }} - persist-credentials: false - #################################################################################################### - # BE VERY CAREFUL HERE! THIS LINE AND THE END OF THE WARNING. HERE WE CHECK OUT THE TARGET - # COMMIT AND ITS PARENT TO BE ABLE TO COMPARE THEM BUT ALSO TO BE ABLE TO BUILD THE IMAGE FROM - # THE INCOMING PR, RATHER THAN FROM TARGET BRANCH. THIS IS A SECURITY RISK, BECAUSE THE PR - # CAN CONTAIN ANY CODE AND WE EXECUTE IT HERE. THEREFORE, WE NEED TO BE VERY CAREFUL WHAT WE - # DO HERE. WE SHOULD NOT EXECUTE ANY CODE THAT COMES FROM THE PR. WE SHOULD NOT RUN ANY BREEZE - # COMMAND NOR SCRIPTS NOR COMPOSITE ACTIONS. WE SHOULD ONLY RUN CODE THAT IS EMBEDDED DIRECTLY IN - # THIS WORKFLOW - BECAUSE THIS IS THE ONLY CODE THAT WE CAN TRUST. - #################################################################################################### - - name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there. - uses: actions/checkout@v4 - with: - path: "target-airflow" - ref: ${{ github.base_ref }} - persist-credentials: false - if: needs.build-info.outputs.is-committer-build != 'true' - - name: > - Replace "scripts/ci", "dev" and ".github/actions" with the target branch - so that the those directories are not coming from the PR - shell: bash - run: | - echo - echo -e "\033[33m Replace scripts, dev, actions with target branch for non-committer builds!\033[0m" - echo - rm -rfv "scripts/ci" - mv -v "target-airflow/scripts/ci" "scripts" - rm -rfv "dev" - mv -v "target-airflow/dev" "." - rm -rfv ".github/actions" - mv -v "target-airflow/.github/actions" ".github" - if: needs.build-info.outputs.is-committer-build != 'true' - #################################################################################################### - # HERE IT'S A BIT SAFER. THE `dev`, `scripts/ci` AND `.github/actions` ARE NOW COMING FROM THE - # BASE_REF - WHICH IS THE TARGET BRANCH OF THE PR. WE CAN TRUST THAT THOSE SCRIPTS ARE SAVE TO RUN. - # ALL THE REST OF THE CODE COMES FROM THE PR, AND FOR EXAMPLE THE CODE IN THE `Dockerfile.ci` CAN - # BE RUN SAFELY AS PART OF DOCKER BUILD. BECAUSE IT RUNS INSIDE THE DOCKER CONTAINER AND IT IS - # ISOLATED FROM THE RUNNER. - #################################################################################################### - - name: Cleanup docker - uses: ./.github/actions/cleanup-docker - - name: "Install Breeze" - uses: ./.github/actions/breeze - - name: Build PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}} - uses: ./.github/actions/build-prod-images - with: - build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} - chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} - python-version: ${{ matrix.python-version }} - push-image: "true" - env: - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} - PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} - DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} + with: + runs-on: ${{ needs.build-info.outputs.runs-on }} + build-type: "Regular" + do-build: ${{ needs.build-info.outputs.ci-image-build }} + target-commit-sha: ${{ needs.build-info.outputs.target-commit-sha }} + pull-request-target: "true" + is-committer-build: ${{ needs.build-info.outputs.is-committer-build }} + push-image: "true" + use-uv: "true" + image-tag: ${{ needs.build-info.outputs.image-tag }} + python-versions: ${{ needs.build-info.outputs.python-versions }} + branch: ${{ needs.build-info.outputs.default-branch }} + constraints-branch: ${{ needs.build-info.outputs.constraints-branch }} + build-provider-packages: "true" + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} + docker-cache: ${{ needs.build-info.outputs.cache-directive }} diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 1b90e010fc98c..16deb0d8145cd 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -32,6 +32,21 @@ on: # yamllint disable-line rule:truthy required: false default: "true" type: string + target-commit-sha: + description: "The commit SHA to checkout for the build" + required: false + default: "" + type: string + pull-request-target: + description: "Whether we are running this from pull-request-target workflow (true/false)" + required: false + default: "false" + type: string + is-committer-build: + description: "Whether the build is executed by committer (true/false)" + required: false + default: "false" + type: string platform: description: > Name of the platform for the build - 'amd64/arm64' @@ -70,14 +85,14 @@ on: # yamllint disable-line rule:truthy description: "Branch used to run the CI jobs in (main/v2_*_test)." required: true type: string - upgrade-to-newer-dependencies: - description: "Whether to attempt to upgrade image to newer dependencies (false/RANDOM_VALUE)" - required: true - type: string constraints-branch: description: "Branch used to construct constraints URL from." required: true type: string + upgrade-to-newer-dependencies: + description: "Whether to attempt to upgrade image to newer dependencies (false/RANDOM_VALUE)" + required: true + type: string docker-cache: description: "Docker cache specification to build the image (registry, local, disabled)." required: true @@ -95,13 +110,14 @@ ${{ inputs.do-build == 'true' && 'Build' || 'Skip building' }} \ CI ${{inputs.platform}} image\ ${{matrix.python-version}}${{ inputs.do-build == 'true' && ':' || '' }}\ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" - runs-on: ${{fromJson(inputs.runs-on)}} + runs-on: ${{ fromJson(inputs.runs-on) }} env: BACKEND: sqlite DEFAULT_BRANCH: ${{ inputs.branch }} DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} VERSION_SUFFIX_FOR_PYPI: "dev0" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + USE_UV: ${{ inputs.use-uv }} steps: - name: "Cleanup repo" shell: bash @@ -109,9 +125,52 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" if: inputs.do-build == 'true' - uses: actions/checkout@v4 with: - ref: ${{ needs.build-info.outputs.targetCommitSha }} + ref: ${{ inputs.target-commit-sha }} persist-credentials: false if: inputs.do-build == 'true' + #################################################################################################### + # BE VERY CAREFUL HERE! THIS LINE AND THE END OF THE WARNING. IN PULL REQUEST TARGET WORKFLOW + # WE CHECK OUT THE TARGET COMMIT ABOVE TO BE ABLE TO BUILD THE IMAGE FROM SOURCES FROM THE + # INCOMING PR, RATHER THAN FROM TARGET BRANCH. THIS IS A SECURITY RISK, BECAUSE THE PR + # CAN CONTAIN ANY CODE AND WE EXECUTE IT HERE. THEREFORE, WE NEED TO BE VERY CAREFUL WHAT WE + # DO HERE. WE SHOULD NOT EXECUTE ANY CODE THAT COMES FROM THE PR. WE SHOULD NOT RUN ANY BREEZE + # COMMAND NOR SCRIPTS NOR COMPOSITE ACTIONS. WE SHOULD ONLY RUN CODE THAT IS EMBEDDED DIRECTLY IN + # THIS WORKFLOW - BECAUSE THIS IS THE ONLY CODE THAT WE CAN TRUST. + #################################################################################################### + - name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there. + uses: actions/checkout@v4 + with: + path: "target-airflow" + ref: ${{ github.base_ref }} + persist-credentials: false + if: > + inputs.do-build == 'true' && inputs.pull-request-target == 'true' && + inputs.is-committer-build != 'true' + - name: > + Replace "scripts/ci", "dev", ".github/actions" and ".github/workflows" with the target branch + so that the those directories are not coming from the PR + shell: bash + run: | + echo + echo -e "\033[33m Replace scripts, dev, actions with target branch for non-committer builds!\033[0m" + echo + rm -rfv "scripts/ci" + rm -rfv "dev" + rm -rfv ".github/actions" + rm -rfv ".github/workflows" + mv -v "target-airflow/scripts/ci" "scripts" + mv -v "target-airflow/dev" "." + mv -v "target-airflow/.github/actions" "target-airflow/.github/workflows" ".github" + if: > + inputs.do-build == 'true' && inputs.pull-request-target == 'true' && + inputs.is-committer-build != 'true' + #################################################################################################### + # HERE IT'S A BIT SAFER. THE `dev`, `scripts/ci` AND `.github/actions` ARE NOW COMING FROM THE + # BASE_REF - WHICH IS THE TARGET BRANCH OF THE PR. WE CAN TRUST THAT THOSE SCRIPTS ARE SAVE TO RUN. + # ALL THE REST OF THE CODE COMES FROM THE PR, AND FOR EXAMPLE THE CODE IN THE `Dockerfile.ci` CAN + # BE RUN SAFELY AS PART OF DOCKER BUILD. BECAUSE IT RUNS INSIDE THE DOCKER CONTAINER AND IT IS + # ISOLATED FROM THE RUNNER. + #################################################################################################### - name: Cleanup docker uses: ./.github/actions/cleanup-docker if: inputs.do-build == 'true' @@ -133,7 +192,6 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" - name: > Build ${{ inputs.push-image == 'true' && ' & push ' || '' }} ${{ inputs.platform }}:${{ matrix.python-version }}:${{ inputs.image-tag }} - shell: bash run: > breeze ci-image build --tag-as-latest --image-tag "${{ inputs.image-tag }}" --python "${{ matrix.python-version }}" @@ -143,7 +201,6 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - USE_UV: ${{ inputs.use-uv }} BUILDER: ${{ inputs.platform == 'amd64' && 'default' || 'airflow_cache' }} PUSH: ${{ inputs.push-image }} if: inputs.do-build == 'true' diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index 2d59ca45b780e..92f8b43b001b3 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -20,6 +20,11 @@ name: Build PROD images on: # yamllint disable-line rule:truthy workflow_call: inputs: + runs-on: + description: "The array of labels (in json form) determining type of the runner to use for the build." + required: false + default: '["ubuntu-22.04"]' + type: string build-type: description: > Name of the 'type' of the build - usually 'Regular' but other types are used to test image @@ -33,6 +38,25 @@ on: # yamllint disable-line rule:truthy required: false default: "true" type: string + target-commit-sha: + description: "The commit SHA to checkout for the build" + required: false + default: "" + type: string + pull-request-target: + description: "Whether we are running this from pull-request-target workflow (true/false)" + required: false + default: "false" + type: string + is-committer-build: + description: "Whether the build is executed by committer (true/false)" + required: false + default: "false" + type: string + push-image: + description: "Whether to push image to the registry (true/false)" + required: true + type: string debian-version: description: "Base Debian distribution to use for the build (bookworm/bullseye)" type: string @@ -49,10 +73,6 @@ on: # yamllint disable-line rule:truthy description: "Tag to set for the image" required: true type: string - push-image: - description: "Whether to push image to the registry (true/false)" - required: true - type: string python-versions: description: "JSON-formatted array of Python versions to build images from" required: true @@ -61,6 +81,10 @@ on: # yamllint disable-line rule:truthy description: "Branch used to run the CI jobs in (main/v2_*_test)." required: true type: string + constraints-branch: + description: "Branch used to construct constraints URL from." + required: true + type: string build-provider-packages: description: "Whether to build provider packages (true/false). If false providers are from PyPI" required: true @@ -73,10 +97,6 @@ on: # yamllint disable-line rule:truthy description: "Space-separated list of providers that should be installed from context files" required: true type: string - constraints-branch: - description: "Branch used to construct constraints URL from." - required: true - type: string docker-cache: description: "Docker cache specification to build the image (registry, local, disabled)." required: true @@ -94,13 +114,15 @@ ${{ inputs.do-build == 'true' && 'Build' || 'Skip building' }} \ PROD ${{inputs.build-type}} image\ ${{matrix.python-version}}${{ inputs.do-build == 'true' && ':' || '' }}\ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" - runs-on: ["ubuntu-22.04"] + runs-on: ${{ fromJSON(inputs.runs-on) }} env: BACKEND: sqlite DEFAULT_BRANCH: ${{ inputs.branch }} DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} VERSION_SUFFIX_FOR_PYPI: "dev0" + INCLUDE_NOT_READY_PROVIDERS: "true" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + USE_UV: ${{ inputs.use-uv }} steps: - name: "Cleanup repo" shell: bash @@ -108,9 +130,52 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" if: inputs.do-build == 'true' - uses: actions/checkout@v4 with: - ref: ${{ needs.build-info.outputs.targetCommitSha }} + ref: ${{ inputs.target-commit-sha }} persist-credentials: false if: inputs.do-build == 'true' + #################################################################################################### + # BE VERY CAREFUL HERE! THIS LINE AND THE END OF THE WARNING. IN PULL REQUEST TARGET WORKFLOW + # WE CHECK OUT THE TARGET COMMIT ABOVE TO BE ABLE TO BUILD THE IMAGE FROM SOURCES FROM THE + # INCOMING PR, RATHER THAN FROM TARGET BRANCH. THIS IS A SECURITY RISK, BECAUSE THE PR + # CAN CONTAIN ANY CODE AND WE EXECUTE IT HERE. THEREFORE, WE NEED TO BE VERY CAREFUL WHAT WE + # DO HERE. WE SHOULD NOT EXECUTE ANY CODE THAT COMES FROM THE PR. WE SHOULD NOT RUN ANY BREEZE + # COMMAND NOR SCRIPTS NOR COMPOSITE ACTIONS. WE SHOULD ONLY RUN CODE THAT IS EMBEDDED DIRECTLY IN + # THIS WORKFLOW - BECAUSE THIS IS THE ONLY CODE THAT WE CAN TRUST. + #################################################################################################### + - name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there. + uses: actions/checkout@v4 + with: + path: "target-airflow" + ref: ${{ github.base_ref }} + persist-credentials: false + if: > + inputs.do-build == 'true' && inputs.pull-request-target == 'true' && + inputs.is-committer-build != 'true' + - name: > + Replace "scripts/ci", "dev", ".github/actions" and ".github/workflows" with the target branch + so that the those directories are not coming from the PR + shell: bash + run: | + echo + echo -e "\033[33m Replace scripts, dev, actions with target branch for non-committer builds!\033[0m" + echo + rm -rfv "scripts/ci" + rm -rfv "dev" + rm -rfv ".github/actions" + rm -rfv ".github/workflows" + mv -v "target-airflow/scripts/ci" "scripts" + mv -v "target-airflow/dev" "." + mv -v "target-airflow/.github/actions" "target-airflow/.github/workflows" ".github" + if: > + inputs.do-build == 'true' && inputs.pull-request-target == 'true' && + inputs.is-committer-build != 'true' + #################################################################################################### + # HERE IT'S A BIT SAFER. THE `dev`, `scripts/ci` AND `.github/actions` ARE NOW COMING FROM THE + # BASE_REF - WHICH IS THE TARGET BRANCH OF THE PR. WE CAN TRUST THAT THOSE SCRIPTS ARE SAVE TO RUN. + # ALL THE REST OF THE CODE COMES FROM THE PR, AND FOR EXAMPLE THE CODE IN THE `Dockerfile.ci` CAN + # BE RUN SAFELY AS PART OF DOCKER BUILD. BECAUSE IT RUNS INSIDE THE DOCKER CONTAINER AND IT IS + # ISOLATED FROM THE RUNNER. + #################################################################################################### - name: Cleanup docker uses: ./.github/actions/cleanup-docker if: inputs.do-build == 'true' @@ -185,7 +250,6 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }} INCLUDE_NOT_READY_PROVIDERS: "true" - USE_UV: ${{ inputs.use-uv }} if: inputs.do-build == 'true' && inputs.build-provider-packages == 'true' - name: "Build PROD images with PyPi providers ${{ matrix.python-version }}:${{ inputs.image-tag }}" shell: bash @@ -201,7 +265,6 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }} INCLUDE_NOT_READY_PROVIDERS: "true" - USE_UV: ${{ inputs.use-uv }} if: inputs.do-build == 'true' && inputs.build-provider-packages != 'true' - name: Verify PROD image ${{ matrix.python-version }}:${{ inputs.image-tag }} run: >