diff --git a/.github/actions/build-ci-images/action.yml b/.github/actions/build-ci-images/action.yml deleted file mode 100644 index b743aad41d307..0000000000000 --- a/.github/actions/build-ci-images/action.yml +++ /dev/null @@ -1,55 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -name: 'Build CI images' -description: 'Build CI images' -inputs: - python-version: - description: 'Python version to use' - required: true -runs: - using: "composite" - steps: - - name: "Install Breeze" - uses: ./.github/actions/breeze - - name: "Check if dependencies are properly regenerated" - shell: bash - run: | - pip install rich>=12.4.4 pyyaml - python scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py - if: env.UPGRADE_TO_NEWER_DEPENDENCIES != 'false' - - name: Login to ghcr.io - shell: bash - run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - name: "Build & Push AMD64 CI images ${{ env.IMAGE_TAG }} ${{ inputs.python-version }}" - shell: bash - run: > - breeze ci-image build --push --tag-as-latest --upgrade-on-failure - --python "${{ inputs.python-version }}" - - name: "Source constraints: ${{ inputs.python-version }}" - shell: bash - run: > - breeze release-management generate-constraints --python "${{ inputs.python-version }}" - --airflow-constraints-mode constraints-source-providers - - name: "Upload constraint artifacts" - uses: actions/upload-artifact@v4 - with: - name: source-constraints-${{ inputs.python-version }} - path: ./files/constraints-*/constraints-source-providers-*.txt - retention-days: 7 - if-no-files-found: error diff --git a/.github/actions/build-prod-images/action.yml b/.github/actions/build-prod-images/action.yml deleted file mode 100644 index 38c79e11beb48..0000000000000 --- a/.github/actions/build-prod-images/action.yml +++ /dev/null @@ -1,108 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -name: 'Build PROD images' -description: 'Build PROD images' -inputs: - python-version: - description: 'Python versions to use' - required: true - type: string - build-provider-packages: - description: 'Whether to build provider packages from sources (true/false)' - required: true - type: string - chicken-egg-providers: - description: 'List space separated chicken-egg provider packages to build from sources' - required: true - type: string - push-image: - description: 'Whether to push image (true/false)' - required: true - type: string -runs: - using: "composite" - steps: - - name: "Regenerate dependencies in case they was modified manually so that we can build an image" - shell: bash - run: | - pip install rich>=12.4.4 pyyaml - python scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py - if: env.UPGRADE_TO_NEWER_DEPENDENCIES != 'false' - - name: "Cleanup dist and context file" - shell: bash - run: rm -fv ./dist/* ./docker-context-files/* - - name: "Prepare providers packages" - shell: bash - run: > - breeze release-management prepare-provider-packages - --package-list-file ./prod_image_installed_providers.txt - --package-format wheel --version-suffix-for-pypi dev0 - if: ${{ inputs.build-provider-packages == 'true' }} - - name: "Prepare chicken-eggs provider packages" - # In case of provider packages which use latest dev0 version of providers, we should prepare them - # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency - # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0 - shell: bash - run: > - breeze release-management prepare-provider-packages - --package-format wheel --version-suffix-for-pypi dev0 ${{ inputs.chicken-egg-providers }} - if: ${{ inputs.build-provider-packages != 'true' && inputs.chicken-egg-providers != '' }} - - name: "Prepare airflow package" - shell: bash - run: > - breeze release-management prepare-airflow-package - --package-format wheel --version-suffix-for-pypi dev0 - - name: "Copy dist packages to docker-context files" - shell: bash - run: cp -v --no-preserve=mode,ownership ./dist/*.whl ./docker-context-files - - name: "Download constraints from the CI build" - uses: actions/download-artifact@v4 - with: - name: source-constraints-${{ inputs.python-version }} - path: ./docker-context-files - if: ${{ inputs.build-provider-packages == 'true' }} - - name: "Download constraints" - uses: actions/download-artifact@v4 - with: - name: constraints - path: ./docker-context-files - if: ${{ inputs.build-provider-packages != 'true' }} - - name: Login to ghcr.io - shell: bash - run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - name: "Build PROD images w/ source providers ${{ inputs.python-version }}:${{ env.IMAGE_TAG }}" - shell: bash - run: > - breeze prod-image build --tag-as-latest - --install-packages-from-context --airflow-constraints-mode constraints-source-providers - --use-constraints-for-context-packages --python "${{ inputs.python-version }}" - env: - COMMIT_SHA: ${{ github.sha }} - PUSH: ${{ inputs.push-image }} - if: ${{ inputs.build-provider-packages == 'true' }} - - name: "Build PROD images with PyPi providers ${{ inputs.python-version }}:${{ env.IMAGE_TAG }}" - shell: bash - run: > - breeze prod-image build --tag-as-latest - --install-packages-from-context --airflow-constraints-mode constraints - --use-constraints-for-context-packages --python "${{ inputs.python-version }}" - env: - COMMIT_SHA: ${{ github.sha }} - PUSH: ${{ inputs.push-image }} - if: ${{ inputs.build-provider-packages != 'true' }} diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 5070c1fa52fae..92f362f5f1cb9 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -51,7 +51,7 @@ jobs: build-info: timeout-minutes: 10 name: "Build Info" - runs-on: 'ubuntu-22.04' + runs-on: ['ubuntu-22.04'] env: TARGET_BRANCH: ${{ github.event.pull_request.base.ref }} outputs: @@ -81,7 +81,7 @@ jobs: github.event.pull_request.head.sha || github.sha }}" - if: github.repository == 'apache/airflow' + # if: github.repository == 'apache/airflow' steps: - name: "Cleanup repo" shell: bash @@ -159,168 +159,60 @@ jobs: GITHUB_CONTEXT: ${{ toJson(github) }} build-ci-images: - strategy: - fail-fast: true - matrix: - python-version: ${{fromJson(needs.build-info.outputs.python-versions)}} + name: Build CI images permissions: contents: read packages: write - timeout-minutes: 80 - name: Build CI image ${{ matrix.python-version }} - runs-on: ["ubuntu-22.04"] + secrets: inherit needs: [build-info] + uses: ./.github/workflows/ci-image-build.yml + # Only run this it if the PR comes from fork, otherwise build will be done "in-PR-workflow" if: | needs.build-info.outputs.ci-image-build == 'true' && github.event.pull_request.head.repo.full_name != 'apache/airflow' - env: - DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} - DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} - RUNS_ON: "${{ needs.build-info.outputs.runs-on }}" - BACKEND: sqlite - VERSION_SUFFIX_FOR_PYPI: "dev0" - USE_UV: "true" - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - uses: actions/checkout@v4 - with: - ref: ${{ needs.build-info.outputs.target-commit-sha }} - persist-credentials: false - #################################################################################################### - # BE VERY CAREFUL HERE! THIS LINE AND THE END OF THE WARNING. HERE WE CHECK OUT THE TARGET - # COMMIT AND ITS PARENT TO BE ABLE TO COMPARE THEM BUT ALSO TO BE ABLE TO BUILD THE IMAGE FROM - # THE INCOMING PR, RATHER THAN FROM TARGET BRANCH. THIS IS A SECURITY RISK, BECAUSE THE PR - # CAN CONTAIN ANY CODE AND WE EXECUTE IT HERE. THEREFORE, WE NEED TO BE VERY CAREFUL WHAT WE - # DO HERE. WE SHOULD NOT EXECUTE ANY CODE THAT COMES FROM THE PR. WE SHOULD NOT RUN ANY BREEZE - # COMMAND NOR SCRIPTS NOR COMPOSITE ACTIONS. WE SHOULD ONLY RUN CODE THAT IS EMBEDDED DIRECTLY IN - # THIS WORKFLOW - BECAUSE THIS IS THE ONLY CODE THAT WE CAN TRUST. - #################################################################################################### - - name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there. - uses: actions/checkout@v4 - with: - path: "target-airflow" - ref: ${{ github.base_ref }} - persist-credentials: false - if: needs.build-info.outputs.is-committer-build != 'true' - - name: > - Replace "scripts/ci", "dev" and ".github/actions" with the target branch - so that the those directories are not coming from the PR - shell: bash - run: | - echo - echo -e "\033[33m Replace scripts, dev, actions with target branch for non-committer builds!\033[0m" - echo - rm -rfv "scripts/ci" - mv -v "target-airflow/scripts/ci" "scripts" - rm -rfv "dev" - mv -v "target-airflow/dev" "." - rm -rfv ".github/actions" - mv -v "target-airflow/.github/actions" ".github" - if: needs.build-info.outputs.is-committer-build != 'true' - #################################################################################################### - # HERE IT'S A BIT SAFER. THE `dev`, `scripts/ci` AND `.github/actions` ARE NOW COMING FROM THE - # BASE_REF - WHICH IS THE TARGET BRANCH OF THE PR. WE CAN TRUST THAT THOSE SCRIPTS ARE SAVE TO RUN. - # ALL THE REST OF THE CODE COMES FROM THE PR, AND FOR EXAMPLE THE CODE IN THE `Dockerfile.ci` CAN - # BE RUN SAFELY AS PART OF DOCKER BUILD. BECAUSE IT RUNS INSIDE THE DOCKER CONTAINER AND IT IS - # ISOLATED FROM THE RUNNER. - #################################################################################################### - - name: Cleanup docker - uses: ./.github/actions/cleanup-docker - - name: Build CI Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}} - uses: ./.github/actions/build-ci-images - with: - python-version: ${{ matrix.python-version }} - env: - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} - PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} - DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} - BUILD_TIMEOUT_MINUTES: 70 + with: + runs-on: ${{ needs.build-info.outputs.runs-on }} + do-build: ${{ needs.build-info.outputs.ci-image-build }} + target-commit-sha: ${{ needs.build-info.outputs.target-commit-sha }} + pull-request-target: "true" + is-committer-build: ${{ needs.build-info.outputs.is-committer-build }} + push-image: "true" + upload-constraints: "true" + use-uv: "true" + image-tag: ${{ needs.build-info.outputs.image-tag }} + python-versions: ${{ needs.build-info.outputs.python-versions }} + branch: ${{ needs.build-info.outputs.default-branch }} + constraints-branch: ${{ needs.build-info.outputs.constraints-branch }} + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + docker-cache: ${{ needs.build-info.outputs.cache-directive }} + build-prod-images: - strategy: - fail-fast: true - matrix: - python-version: ${{fromJson(needs.build-info.outputs.python-versions)}} + name: Build PROD images permissions: contents: read packages: write - timeout-minutes: 80 - name: Build PROD image ${{ matrix.python-version }} - runs-on: ["ubuntu-22.04"] - needs: [build-info, build-ci-images] + secrets: inherit + needs: [build-info] + uses: ./.github/workflows/prod-image-build.yml + # Only run this it if the PR comes from fork, otherwise build will be done "in-PR-workflow" if: | needs.build-info.outputs.prod-image-build == 'true' && github.event.pull_request.head.repo.full_name != 'apache/airflow' - env: - DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} - DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} - RUNS_ON: "${{ needs.build-info.outputs.runs-on }}" - BACKEND: sqlite - VERSION_SUFFIX_FOR_PYPI: "dev0" - INCLUDE_NOT_READY_PROVIDERS: "true" - USE_UV: "true" - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - uses: actions/checkout@v4 - with: - ref: ${{ needs.build-info.outputs.target-commit-sha }} - persist-credentials: false - #################################################################################################### - # BE VERY CAREFUL HERE! THIS LINE AND THE END OF THE WARNING. HERE WE CHECK OUT THE TARGET - # COMMIT AND ITS PARENT TO BE ABLE TO COMPARE THEM BUT ALSO TO BE ABLE TO BUILD THE IMAGE FROM - # THE INCOMING PR, RATHER THAN FROM TARGET BRANCH. THIS IS A SECURITY RISK, BECAUSE THE PR - # CAN CONTAIN ANY CODE AND WE EXECUTE IT HERE. THEREFORE, WE NEED TO BE VERY CAREFUL WHAT WE - # DO HERE. WE SHOULD NOT EXECUTE ANY CODE THAT COMES FROM THE PR. WE SHOULD NOT RUN ANY BREEZE - # COMMAND NOR SCRIPTS NOR COMPOSITE ACTIONS. WE SHOULD ONLY RUN CODE THAT IS EMBEDDED DIRECTLY IN - # THIS WORKFLOW - BECAUSE THIS IS THE ONLY CODE THAT WE CAN TRUST. - #################################################################################################### - - name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there. - uses: actions/checkout@v4 - with: - path: "target-airflow" - ref: ${{ github.base_ref }} - persist-credentials: false - if: needs.build-info.outputs.is-committer-build != 'true' - - name: > - Replace "scripts/ci", "dev" and ".github/actions" with the target branch - so that the those directories are not coming from the PR - shell: bash - run: | - echo - echo -e "\033[33m Replace scripts, dev, actions with target branch for non-committer builds!\033[0m" - echo - rm -rfv "scripts/ci" - mv -v "target-airflow/scripts/ci" "scripts" - rm -rfv "dev" - mv -v "target-airflow/dev" "." - rm -rfv ".github/actions" - mv -v "target-airflow/.github/actions" ".github" - if: needs.build-info.outputs.is-committer-build != 'true' - #################################################################################################### - # HERE IT'S A BIT SAFER. THE `dev`, `scripts/ci` AND `.github/actions` ARE NOW COMING FROM THE - # BASE_REF - WHICH IS THE TARGET BRANCH OF THE PR. WE CAN TRUST THAT THOSE SCRIPTS ARE SAVE TO RUN. - # ALL THE REST OF THE CODE COMES FROM THE PR, AND FOR EXAMPLE THE CODE IN THE `Dockerfile.ci` CAN - # BE RUN SAFELY AS PART OF DOCKER BUILD. BECAUSE IT RUNS INSIDE THE DOCKER CONTAINER AND IT IS - # ISOLATED FROM THE RUNNER. - #################################################################################################### - - name: Cleanup docker - uses: ./.github/actions/cleanup-docker - - name: "Install Breeze" - uses: ./.github/actions/breeze - - name: Build PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}} - uses: ./.github/actions/build-prod-images - with: - build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} - chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} - python-version: ${{ matrix.python-version }} - push-image: "true" - env: - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} - PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} - DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} + with: + runs-on: ${{ needs.build-info.outputs.runs-on }} + build-type: "Regular" + do-build: ${{ needs.build-info.outputs.ci-image-build }} + target-commit-sha: ${{ needs.build-info.outputs.target-commit-sha }} + pull-request-target: "true" + is-committer-build: ${{ needs.build-info.outputs.is-committer-build }} + push-image: "true" + use-uv: "true" + image-tag: ${{ needs.build-info.outputs.image-tag }} + python-versions: ${{ needs.build-info.outputs.python-versions }} + branch: ${{ needs.build-info.outputs.default-branch }} + constraints-branch: ${{ needs.build-info.outputs.constraints-branch }} + build-provider-packages: "true" + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} + docker-cache: ${{ needs.build-info.outputs.cache-directive }} diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 1b90e010fc98c..5067557d572d9 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -17,6 +17,7 @@ # --- name: Build CI images +xxx on: # yamllint disable-line rule:truthy workflow_call: inputs: @@ -32,6 +33,21 @@ on: # yamllint disable-line rule:truthy required: false default: "true" type: string + target-commit-sha: + description: "The commit SHA to checkout for the build" + required: false + default: "" + type: string + pull-request-target: + description: "Whether we are running this from pull-request-target workflow (true/false)" + required: false + default: "false" + type: string + is-committer-build: + description: "Whether the build is executed by committer (true/false)" + required: false + default: "false" + type: string platform: description: > Name of the platform for the build - 'amd64/arm64' @@ -70,14 +86,14 @@ on: # yamllint disable-line rule:truthy description: "Branch used to run the CI jobs in (main/v2_*_test)." required: true type: string - upgrade-to-newer-dependencies: - description: "Whether to attempt to upgrade image to newer dependencies (false/RANDOM_VALUE)" - required: true - type: string constraints-branch: description: "Branch used to construct constraints URL from." required: true type: string + upgrade-to-newer-dependencies: + description: "Whether to attempt to upgrade image to newer dependencies (false/RANDOM_VALUE)" + required: true + type: string docker-cache: description: "Docker cache specification to build the image (registry, local, disabled)." required: true @@ -95,13 +111,14 @@ ${{ inputs.do-build == 'true' && 'Build' || 'Skip building' }} \ CI ${{inputs.platform}} image\ ${{matrix.python-version}}${{ inputs.do-build == 'true' && ':' || '' }}\ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" - runs-on: ${{fromJson(inputs.runs-on)}} + runs-on: ${{ fromJson(inputs.runs-on) }} env: BACKEND: sqlite DEFAULT_BRANCH: ${{ inputs.branch }} DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} VERSION_SUFFIX_FOR_PYPI: "dev0" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + USE_UV: ${{ inputs.use-uv }} steps: - name: "Cleanup repo" shell: bash @@ -109,9 +126,52 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" if: inputs.do-build == 'true' - uses: actions/checkout@v4 with: - ref: ${{ needs.build-info.outputs.targetCommitSha }} + ref: ${{ inputs.target-commit-sha }} persist-credentials: false if: inputs.do-build == 'true' + #################################################################################################### + # BE VERY CAREFUL HERE! THIS LINE AND THE END OF THE WARNING. IN PULL REQUEST TARGET WORKFLOW + # WE CHECK OUT THE TARGET COMMIT ABOVE TO BE ABLE TO BUILD THE IMAGE FROM SOURCES FROM THE + # INCOMING PR, RATHER THAN FROM TARGET BRANCH. THIS IS A SECURITY RISK, BECAUSE THE PR + # CAN CONTAIN ANY CODE AND WE EXECUTE IT HERE. THEREFORE, WE NEED TO BE VERY CAREFUL WHAT WE + # DO HERE. WE SHOULD NOT EXECUTE ANY CODE THAT COMES FROM THE PR. WE SHOULD NOT RUN ANY BREEZE + # COMMAND NOR SCRIPTS NOR COMPOSITE ACTIONS. WE SHOULD ONLY RUN CODE THAT IS EMBEDDED DIRECTLY IN + # THIS WORKFLOW - BECAUSE THIS IS THE ONLY CODE THAT WE CAN TRUST. + #################################################################################################### + - name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there. + uses: actions/checkout@v4 + with: + path: "target-airflow" + ref: ${{ github.base_ref }} + persist-credentials: false + if: > + inputs.do-build == 'true' && inputs.pull-request-target == 'true' && + inputs.is-committer-build != 'true' + - name: > + Replace "scripts/ci", "dev", ".github/actions" and ".github/workflows" with the target branch + so that the those directories are not coming from the PR + shell: bash + run: | + echo + echo -e "\033[33m Replace scripts, dev, actions with target branch for non-committer builds!\033[0m" + echo + rm -rfv "scripts/ci" + rm -rfv "dev" + rm -rfv ".github/actions" + rm -rfv ".github/workflows" + mv -v "target-airflow/scripts/ci" "scripts" + mv -v "target-airflow/dev" "." + mv -v "target-airflow/.github/actions" "target-airflow/.github/workflows" ".github" + if: > + inputs.do-build == 'true' && inputs.pull-request-target == 'true' && + inputs.is-committer-build != 'true' + #################################################################################################### + # HERE IT'S A BIT SAFER. THE `dev`, `scripts/ci` AND `.github/actions` ARE NOW COMING FROM THE + # BASE_REF - WHICH IS THE TARGET BRANCH OF THE PR. WE CAN TRUST THAT THOSE SCRIPTS ARE SAVE TO RUN. + # ALL THE REST OF THE CODE COMES FROM THE PR, AND FOR EXAMPLE THE CODE IN THE `Dockerfile.ci` CAN + # BE RUN SAFELY AS PART OF DOCKER BUILD. BECAUSE IT RUNS INSIDE THE DOCKER CONTAINER AND IT IS + # ISOLATED FROM THE RUNNER. + #################################################################################################### - name: Cleanup docker uses: ./.github/actions/cleanup-docker if: inputs.do-build == 'true' @@ -133,7 +193,6 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" - name: > Build ${{ inputs.push-image == 'true' && ' & push ' || '' }} ${{ inputs.platform }}:${{ matrix.python-version }}:${{ inputs.image-tag }} - shell: bash run: > breeze ci-image build --tag-as-latest --image-tag "${{ inputs.image-tag }}" --python "${{ matrix.python-version }}" @@ -143,7 +202,6 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - USE_UV: ${{ inputs.use-uv }} BUILDER: ${{ inputs.platform == 'amd64' && 'default' || 'airflow_cache' }} PUSH: ${{ inputs.push-image }} if: inputs.do-build == 'true' diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index 2d59ca45b780e..92f8b43b001b3 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -20,6 +20,11 @@ name: Build PROD images on: # yamllint disable-line rule:truthy workflow_call: inputs: + runs-on: + description: "The array of labels (in json form) determining type of the runner to use for the build." + required: false + default: '["ubuntu-22.04"]' + type: string build-type: description: > Name of the 'type' of the build - usually 'Regular' but other types are used to test image @@ -33,6 +38,25 @@ on: # yamllint disable-line rule:truthy required: false default: "true" type: string + target-commit-sha: + description: "The commit SHA to checkout for the build" + required: false + default: "" + type: string + pull-request-target: + description: "Whether we are running this from pull-request-target workflow (true/false)" + required: false + default: "false" + type: string + is-committer-build: + description: "Whether the build is executed by committer (true/false)" + required: false + default: "false" + type: string + push-image: + description: "Whether to push image to the registry (true/false)" + required: true + type: string debian-version: description: "Base Debian distribution to use for the build (bookworm/bullseye)" type: string @@ -49,10 +73,6 @@ on: # yamllint disable-line rule:truthy description: "Tag to set for the image" required: true type: string - push-image: - description: "Whether to push image to the registry (true/false)" - required: true - type: string python-versions: description: "JSON-formatted array of Python versions to build images from" required: true @@ -61,6 +81,10 @@ on: # yamllint disable-line rule:truthy description: "Branch used to run the CI jobs in (main/v2_*_test)." required: true type: string + constraints-branch: + description: "Branch used to construct constraints URL from." + required: true + type: string build-provider-packages: description: "Whether to build provider packages (true/false). If false providers are from PyPI" required: true @@ -73,10 +97,6 @@ on: # yamllint disable-line rule:truthy description: "Space-separated list of providers that should be installed from context files" required: true type: string - constraints-branch: - description: "Branch used to construct constraints URL from." - required: true - type: string docker-cache: description: "Docker cache specification to build the image (registry, local, disabled)." required: true @@ -94,13 +114,15 @@ ${{ inputs.do-build == 'true' && 'Build' || 'Skip building' }} \ PROD ${{inputs.build-type}} image\ ${{matrix.python-version}}${{ inputs.do-build == 'true' && ':' || '' }}\ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" - runs-on: ["ubuntu-22.04"] + runs-on: ${{ fromJSON(inputs.runs-on) }} env: BACKEND: sqlite DEFAULT_BRANCH: ${{ inputs.branch }} DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} VERSION_SUFFIX_FOR_PYPI: "dev0" + INCLUDE_NOT_READY_PROVIDERS: "true" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + USE_UV: ${{ inputs.use-uv }} steps: - name: "Cleanup repo" shell: bash @@ -108,9 +130,52 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" if: inputs.do-build == 'true' - uses: actions/checkout@v4 with: - ref: ${{ needs.build-info.outputs.targetCommitSha }} + ref: ${{ inputs.target-commit-sha }} persist-credentials: false if: inputs.do-build == 'true' + #################################################################################################### + # BE VERY CAREFUL HERE! THIS LINE AND THE END OF THE WARNING. IN PULL REQUEST TARGET WORKFLOW + # WE CHECK OUT THE TARGET COMMIT ABOVE TO BE ABLE TO BUILD THE IMAGE FROM SOURCES FROM THE + # INCOMING PR, RATHER THAN FROM TARGET BRANCH. THIS IS A SECURITY RISK, BECAUSE THE PR + # CAN CONTAIN ANY CODE AND WE EXECUTE IT HERE. THEREFORE, WE NEED TO BE VERY CAREFUL WHAT WE + # DO HERE. WE SHOULD NOT EXECUTE ANY CODE THAT COMES FROM THE PR. WE SHOULD NOT RUN ANY BREEZE + # COMMAND NOR SCRIPTS NOR COMPOSITE ACTIONS. WE SHOULD ONLY RUN CODE THAT IS EMBEDDED DIRECTLY IN + # THIS WORKFLOW - BECAUSE THIS IS THE ONLY CODE THAT WE CAN TRUST. + #################################################################################################### + - name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there. + uses: actions/checkout@v4 + with: + path: "target-airflow" + ref: ${{ github.base_ref }} + persist-credentials: false + if: > + inputs.do-build == 'true' && inputs.pull-request-target == 'true' && + inputs.is-committer-build != 'true' + - name: > + Replace "scripts/ci", "dev", ".github/actions" and ".github/workflows" with the target branch + so that the those directories are not coming from the PR + shell: bash + run: | + echo + echo -e "\033[33m Replace scripts, dev, actions with target branch for non-committer builds!\033[0m" + echo + rm -rfv "scripts/ci" + rm -rfv "dev" + rm -rfv ".github/actions" + rm -rfv ".github/workflows" + mv -v "target-airflow/scripts/ci" "scripts" + mv -v "target-airflow/dev" "." + mv -v "target-airflow/.github/actions" "target-airflow/.github/workflows" ".github" + if: > + inputs.do-build == 'true' && inputs.pull-request-target == 'true' && + inputs.is-committer-build != 'true' + #################################################################################################### + # HERE IT'S A BIT SAFER. THE `dev`, `scripts/ci` AND `.github/actions` ARE NOW COMING FROM THE + # BASE_REF - WHICH IS THE TARGET BRANCH OF THE PR. WE CAN TRUST THAT THOSE SCRIPTS ARE SAVE TO RUN. + # ALL THE REST OF THE CODE COMES FROM THE PR, AND FOR EXAMPLE THE CODE IN THE `Dockerfile.ci` CAN + # BE RUN SAFELY AS PART OF DOCKER BUILD. BECAUSE IT RUNS INSIDE THE DOCKER CONTAINER AND IT IS + # ISOLATED FROM THE RUNNER. + #################################################################################################### - name: Cleanup docker uses: ./.github/actions/cleanup-docker if: inputs.do-build == 'true' @@ -185,7 +250,6 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }} INCLUDE_NOT_READY_PROVIDERS: "true" - USE_UV: ${{ inputs.use-uv }} if: inputs.do-build == 'true' && inputs.build-provider-packages == 'true' - name: "Build PROD images with PyPi providers ${{ matrix.python-version }}:${{ inputs.image-tag }}" shell: bash @@ -201,7 +265,6 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }} INCLUDE_NOT_READY_PROVIDERS: "true" - USE_UV: ${{ inputs.use-uv }} if: inputs.do-build == 'true' && inputs.build-provider-packages != 'true' - name: Verify PROD image ${{ matrix.python-version }}:${{ inputs.image-tag }} run: >