From a03324382b2f891fbcecf5ff637213a08e9b877b Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 27 Mar 2024 01:57:07 +0100 Subject: [PATCH] Improve finalization of tests (#38499) Last step of the test preparation is pushing the latest images for AMD to github registry. This should happen using the cache of ours so we should use the `airflow_cache` builders to make it really fast. For PROD image we also want to run them in parallel, because this way we will reuse provider packages and airflow packages built earlier in the same job. Also while we are touching builders, we remove last place where builders were passed by env variables. Co-authored-by: Idan Tepper <65827206+idantepper@users.noreply.github.com> --- .github/workflows/ci-image-build.yml | 6 ++---- .github/workflows/finalize-tests.yml | 9 +++++---- .github/workflows/prod-image-build.yml | 5 ++--- .github/workflows/push-image-cache.yml | 12 ++++++------ 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 177cd39fbb342..e384d7cf837e6 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -192,15 +192,13 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" Build ${{ inputs.push-image == 'true' && ' & push ' || '' }} ${{ inputs.platform }}:${{ matrix.python-version }}:${{ inputs.image-tag }} run: > - breeze ci-image build --tag-as-latest --image-tag "${{ inputs.image-tag }}" - --python "${{ matrix.python-version }}" - --platform "${{ inputs.platform }}" + breeze ci-image build --builder airflow_cache --tag-as-latest --image-tag "${{ inputs.image-tag }}" + --python "${{ matrix.python-version }}" --platform "${{ inputs.platform }}" env: DOCKER_CACHE: ${{ inputs.docker-cache }} INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BUILDER: "airflow_cache" PUSH: ${{ inputs.push-image }} VERBOSE: "true" if: inputs.do-build == 'true' diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index a10b1826d1b57..52c451545beee 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -121,7 +121,7 @@ jobs: git push push-buildx-cache-to-github-registry-amd: - name: Push Regular Image Cache + name: Push Regular AMD Image Cache needs: [update-constraints] uses: ./.github/workflows/push-image-cache.yml permissions: @@ -129,7 +129,8 @@ jobs: packages: write secrets: inherit with: - cache-type: "Regular" + runs-on: '["ubuntu-22.04"]' + cache-type: "Regular AMD" include-prod-images: "true" push-latest-images: "true" platform: "linux/amd64" @@ -142,7 +143,7 @@ jobs: if: inputs.canary-run == 'true' push-buildx-cache-to-github-registry-arm: - name: Push Regular Image Cache + name: Push Regular ARM Image Cache needs: [update-constraints] uses: ./.github/workflows/push-image-cache.yml permissions: @@ -151,7 +152,7 @@ jobs: secrets: inherit with: runs-on: ${{ inputs.runs-on }} - cache-type: "Regular" + cache-type: "Regular ARM" include-prod-images: "true" push-latest-images: "true" platform: "linux/arm64" diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index 27cd62d42c0a7..9556633c40e68 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -252,7 +252,6 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" PUSH: ${{ inputs.push-image }} DOCKER_CACHE: ${{ inputs.docker-cache }} DEBIAN_VERSION: ${{ inputs.debian-version }} - BUILDER: "airflow_cache" INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }} INCLUDE_NOT_READY_PROVIDERS: "true" @@ -260,8 +259,8 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" - name: "Build PROD images with PyPi providers ${{ matrix.python-version }}:${{ inputs.image-tag }}" shell: bash run: > - breeze prod-image build --tag-as-latest --image-tag "${{ inputs.image-tag }}" - --commit-sha "${{ github.sha }}" + breeze prod-image build --builder airflow_cache --tag-as-latest + --image-tag "${{ inputs.image-tag }}" --commit-sha "${{ github.sha }}" --install-packages-from-context --airflow-constraints-mode constraints --use-constraints-for-context-packages --python "${{ matrix.python-version }}" env: diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index c89ef49302204..186b2f09878d0 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -74,8 +74,8 @@ on: # yamllint disable-line rule:truthy required: true type: string jobs: - push-image-cache: - name: "Push CI ${{ inputs.cache-type }} image cache" + push-ci-image-cache: + name: "Push CI ${{ inputs.cache-type }}:${{ matrix.python }} image cache " runs-on: ${{ fromJSON(inputs.runs-on) }} strategy: fail-fast: false @@ -120,12 +120,12 @@ jobs: if: always() && inputs.platform == 'linux/arm64' - name: "Push CI latest images: ${{ matrix.python }} (linux/amd64 only)" run: > - breeze ci-image build --push + breeze ci-image build --builder airflow_cache --push --python "${{ matrix.python }}" --platform "${{ inputs.platform }}" if: inputs.push-latest-images == 'true' && inputs.platform == 'linux/amd64' push-prod-image-cache: - name: "Push PROD ${{ inputs.cache-type }} image cache" + name: "Push PROD ${{ inputs.cache-type }}:${{ matrix.python }} image cache" runs-on: ${{ fromJSON(inputs.runs-on) }} strategy: fail-fast: false @@ -186,6 +186,6 @@ jobs: # and we can do it after we stopped the ARM instance as it is not needed anymore - name: "Push PROD latest image: ${{ matrix.python }} (linux/amd64 ONLY)" run: > - breeze prod-image build --install-packages-from-context - --push --python ${{ matrix.python}} --platform "${{ inputs.platform }}" + breeze prod-image build --builder airflow_cache --install-packages-from-context + --push --platform "${{ inputs.platform }}" if: inputs.push-latest-images == 'true' && inputs.platform == 'linux/amd64'