Update ci-image-build.yml #1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | ||
| # or more contributor license agreements. See the NOTICE file | ||
| # distributed with this work for additional information | ||
| # regarding copyright ownership. The ASF licenses this file | ||
| # to you under the Apache License, Version 2.0 (the | ||
| # "License"); you may not use this file except in compliance | ||
| # with the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, | ||
| # software distributed under the License is distributed on an | ||
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| # KIND, either express or implied. See the License for the | ||
| # specific language governing permissions and limitations | ||
| # under the License. | ||
| # | ||
| --- | ||
| name: Build CI images | ||
| xxx | ||
| on: # yamllint disable-line rule:truthy | ||
| workflow_call: | ||
| inputs: | ||
| runs-on: | ||
| description: "The array of labels (in json form) determining type of the runner to use for the build." | ||
| required: false | ||
| default: '["ubuntu-22.04"]' | ||
| type: string | ||
| do-build: | ||
| description: > | ||
| Whether to actually do the build (true/false). If set to false, the build is done | ||
| already in pull-request-target workflow, so we skip it here. | ||
| required: false | ||
| default: "true" | ||
| type: string | ||
| target-commit-sha: | ||
| description: "The commit SHA to checkout for the build" | ||
| required: false | ||
| default: "" | ||
| type: string | ||
| pull-request-target: | ||
| description: "Whether we are running this from pull-request-target workflow (true/false)" | ||
| required: false | ||
| default: "false" | ||
| type: string | ||
| is-committer-build: | ||
| description: "Whether the build is executed by committer (true/false)" | ||
| required: false | ||
| default: "false" | ||
| type: string | ||
| platform: | ||
| description: > | ||
| Name of the platform for the build - 'amd64/arm64' | ||
| default: "amd64" | ||
| type: string | ||
| push-image: | ||
| description: "Whether to push image to the registry (true/false)" | ||
| required: false | ||
| default: "true" | ||
| type: string | ||
| upload-constraints: | ||
| description: "Whether to upload constraints artifacts" | ||
| default: "false" | ||
| type: string | ||
| debian-version: | ||
| description: "Base Debian distribution to use for the build (bookworm/bullseye)" | ||
| type: string | ||
| default: "bookworm" | ||
| install-mysql-client-type: | ||
| description: "MySQL client type to use during build (mariadb/mysql)" | ||
| type: string | ||
| default: "mariadb" | ||
| use-uv: | ||
| description: "Whether to use uv to build the image (true/false)" | ||
| required: true | ||
| type: string | ||
| image-tag: | ||
| description: "Tag to set for the image" | ||
| required: true | ||
| type: string | ||
| python-versions: | ||
| description: "JSON-formatted array of Python versions to build images from" | ||
| required: true | ||
| type: string | ||
| branch: | ||
| description: "Branch used to run the CI jobs in (main/v2_*_test)." | ||
| required: true | ||
| type: string | ||
| constraints-branch: | ||
| description: "Branch used to construct constraints URL from." | ||
| required: true | ||
| type: string | ||
| upgrade-to-newer-dependencies: | ||
| description: "Whether to attempt to upgrade image to newer dependencies (false/RANDOM_VALUE)" | ||
| required: true | ||
| type: string | ||
| docker-cache: | ||
| description: "Docker cache specification to build the image (registry, local, disabled)." | ||
| required: true | ||
| type: string | ||
| jobs: | ||
| build-ci-images: | ||
| strategy: | ||
| fail-fast: true | ||
| matrix: | ||
| # yamllint disable-line rule:line-length | ||
| python-version: ${{ inputs.do-build == 'true' && fromJson(inputs.python-versions) || fromJson('[""]') }} | ||
| timeout-minutes: 110 | ||
| name: "\ | ||
| ${{ inputs.do-build == 'true' && 'Build' || 'Skip building' }} \ | ||
| CI ${{inputs.platform}} image\ | ||
| ${{matrix.python-version}}${{ inputs.do-build == 'true' && ':' || '' }}\ | ||
| ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" | ||
| runs-on: ${{ fromJson(inputs.runs-on) }} | ||
| env: | ||
| BACKEND: sqlite | ||
| DEFAULT_BRANCH: ${{ inputs.branch }} | ||
| DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} | ||
| VERSION_SUFFIX_FOR_PYPI: "dev0" | ||
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
| USE_UV: ${{ inputs.use-uv }} | ||
| steps: | ||
| - name: "Cleanup repo" | ||
| shell: bash | ||
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | ||
| if: inputs.do-build == 'true' | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| ref: ${{ inputs.target-commit-sha }} | ||
| persist-credentials: false | ||
| if: inputs.do-build == 'true' | ||
| #################################################################################################### | ||
| # BE VERY CAREFUL HERE! THIS LINE AND THE END OF THE WARNING. IN PULL REQUEST TARGET WORKFLOW | ||
| # WE CHECK OUT THE TARGET COMMIT ABOVE TO BE ABLE TO BUILD THE IMAGE FROM SOURCES FROM THE | ||
| # INCOMING PR, RATHER THAN FROM TARGET BRANCH. THIS IS A SECURITY RISK, BECAUSE THE PR | ||
| # CAN CONTAIN ANY CODE AND WE EXECUTE IT HERE. THEREFORE, WE NEED TO BE VERY CAREFUL WHAT WE | ||
| # DO HERE. WE SHOULD NOT EXECUTE ANY CODE THAT COMES FROM THE PR. WE SHOULD NOT RUN ANY BREEZE | ||
| # COMMAND NOR SCRIPTS NOR COMPOSITE ACTIONS. WE SHOULD ONLY RUN CODE THAT IS EMBEDDED DIRECTLY IN | ||
| # THIS WORKFLOW - BECAUSE THIS IS THE ONLY CODE THAT WE CAN TRUST. | ||
| #################################################################################################### | ||
| - name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there. | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| path: "target-airflow" | ||
| ref: ${{ github.base_ref }} | ||
| persist-credentials: false | ||
| if: > | ||
| inputs.do-build == 'true' && inputs.pull-request-target == 'true' && | ||
| inputs.is-committer-build != 'true' | ||
| - name: > | ||
| Replace "scripts/ci", "dev", ".github/actions" and ".github/workflows" with the target branch | ||
| so that the those directories are not coming from the PR | ||
| shell: bash | ||
| run: | | ||
| echo | ||
| echo -e "\033[33m Replace scripts, dev, actions with target branch for non-committer builds!\033[0m" | ||
| echo | ||
| rm -rfv "scripts/ci" | ||
| rm -rfv "dev" | ||
| rm -rfv ".github/actions" | ||
| rm -rfv ".github/workflows" | ||
| mv -v "target-airflow/scripts/ci" "scripts" | ||
| mv -v "target-airflow/dev" "." | ||
| mv -v "target-airflow/.github/actions" "target-airflow/.github/workflows" ".github" | ||
| if: > | ||
| inputs.do-build == 'true' && inputs.pull-request-target == 'true' && | ||
| inputs.is-committer-build != 'true' | ||
| #################################################################################################### | ||
| # HERE IT'S A BIT SAFER. THE `dev`, `scripts/ci` AND `.github/actions` ARE NOW COMING FROM THE | ||
| # BASE_REF - WHICH IS THE TARGET BRANCH OF THE PR. WE CAN TRUST THAT THOSE SCRIPTS ARE SAVE TO RUN. | ||
| # ALL THE REST OF THE CODE COMES FROM THE PR, AND FOR EXAMPLE THE CODE IN THE `Dockerfile.ci` CAN | ||
| # BE RUN SAFELY AS PART OF DOCKER BUILD. BECAUSE IT RUNS INSIDE THE DOCKER CONTAINER AND IT IS | ||
| # ISOLATED FROM THE RUNNER. | ||
| #################################################################################################### | ||
| - name: Cleanup docker | ||
| uses: ./.github/actions/cleanup-docker | ||
| if: inputs.do-build == 'true' | ||
| - name: "Install Breeze" | ||
| uses: ./.github/actions/breeze | ||
| if: inputs.do-build == 'true' | ||
| - name: "Regenerate dependencies in case they was modified manually so that we can build an image" | ||
| shell: bash | ||
| run: | | ||
| pip install rich>=12.4.4 pyyaml | ||
| python scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py | ||
| if: inputs.do-build == 'true' && inputs.upgrade-to-newer-dependencies != 'false' | ||
| - name: "Start ARM instance" | ||
| run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh | ||
| if: inputs.do-build == 'true' && inputs.platform == 'arm64' | ||
| - name: Login to ghcr.io | ||
| run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin | ||
| if: inputs.do-build == 'true' | ||
| - name: > | ||
| Build ${{ inputs.push-image == 'true' && ' & push ' || '' }} | ||
| ${{ inputs.platform }}:${{ matrix.python-version }}:${{ inputs.image-tag }} | ||
| run: > | ||
| breeze ci-image build --tag-as-latest --image-tag "${{ inputs.image-tag }}" | ||
| --python "${{ matrix.python-version }}" | ||
| --platform "linux/${{ inputs.platform }}" | ||
| env: | ||
| DOCKER_CACHE: ${{ inputs.cache-directive }} | ||
| INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} | ||
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }} | ||
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
| BUILDER: ${{ inputs.platform == 'amd64' && 'default' || 'airflow_cache' }} | ||
| PUSH: ${{ inputs.push-image }} | ||
| if: inputs.do-build == 'true' | ||
| - name: "Stop ARM instance" | ||
| run: ./scripts/ci/images/ci_stop_arm_instance.sh | ||
| if: always() && inputs.do-build == 'true' && inputs.platform == 'arm64' | ||
| - name: "Source constraints: ${{ matrix.python-version }}" | ||
| shell: bash | ||
| run: > | ||
| breeze release-management generate-constraints --python "${{ matrix.python-version }}" | ||
| --airflow-constraints-mode constraints-source-providers --answer yes | ||
| if: inputs.do-build == 'true' && inputs.upload-constraints == 'true' | ||
| - name: "Upload constraint artifacts" | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: source-constraints-${{ matrix.python-version }} | ||
| path: ./files/constraints-*/constraints-source-providers-*.txt | ||
| retention-days: 7 | ||
| if-no-files-found: error | ||
| if: inputs.do-build == 'true' && inputs.upload-constraints == 'true' | ||
