Improve finalization of tests (#38499)

Last step of the test preparation is pushing the latest images for
AMD to github registry. This should happen using the cache of ours
so we should use the `airflow_cache` builders to make it really fast.

For PROD image we also want to run them in parallel, because this
way we will reuse provider packages and airflow packages
built earlier in the same job.

Also while we are touching builders, we remove last place where
builders were passed by env variables.

Co-authored-by: Idan Tepper <[email protected]>

.github/workflows/ci-image-build.yml

@@ -192,15 +192,13 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
           Build ${{ inputs.push-image == 'true' && ' & push ' || '' }}
           ${{ inputs.platform }}:${{ matrix.python-version }}:${{ inputs.image-tag }}
         run: >
-          breeze ci-image build --tag-as-latest --image-tag "${{ inputs.image-tag }}"
-          --python "${{ matrix.python-version }}"
-          --platform "${{ inputs.platform }}"
+          breeze ci-image build --builder airflow_cache --tag-as-latest --image-tag "${{ inputs.image-tag }}"
+          --python "${{ matrix.python-version }}" --platform "${{ inputs.platform }}"
         env:
           DOCKER_CACHE: ${{ inputs.docker-cache }}
           INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          BUILDER: "airflow_cache"
           PUSH: ${{ inputs.push-image }}
           VERBOSE: "true"
         if: inputs.do-build == 'true'

.github/workflows/finalize-tests.yml

@@ -121,15 +121,16 @@ jobs:
           git push
 
   push-buildx-cache-to-github-registry-amd:
-    name: Push Regular Image Cache
+    name: Push Regular AMD Image Cache
     needs: [update-constraints]
     uses: ./.github/workflows/push-image-cache.yml
     permissions:
       contents: read
       packages: write
     secrets: inherit
     with:
-      cache-type: "Regular"
+      runs-on: '["ubuntu-22.04"]'
+      cache-type: "Regular AMD"
       include-prod-images: "true"
       push-latest-images: "true"
       platform: "linux/amd64"
@@ -142,7 +143,7 @@ jobs:
     if: inputs.canary-run == 'true'
 
   push-buildx-cache-to-github-registry-arm:
-    name: Push Regular Image Cache
+    name: Push Regular ARM Image Cache
     needs: [update-constraints]
     uses: ./.github/workflows/push-image-cache.yml
     permissions:
@@ -151,7 +152,7 @@ jobs:
     secrets: inherit
     with:
       runs-on: ${{ inputs.runs-on }}
-      cache-type: "Regular"
+      cache-type: "Regular ARM"
       include-prod-images: "true"
       push-latest-images: "true"
       platform: "linux/arm64"

.github/workflows/prod-image-build.yml

@@ -252,16 +252,15 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
           PUSH: ${{ inputs.push-image }}
           DOCKER_CACHE: ${{ inputs.docker-cache }}
           DEBIAN_VERSION: ${{ inputs.debian-version }}
-          BUILDER: "airflow_cache"
           INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }}
           INCLUDE_NOT_READY_PROVIDERS: "true"
         if: inputs.do-build == 'true' && inputs.build-provider-packages == 'true'
       - name: "Build PROD images with PyPi providers ${{ matrix.python-version }}:${{ inputs.image-tag }}"
         shell: bash
         run: >
-          breeze prod-image build --tag-as-latest --image-tag "${{ inputs.image-tag }}"
-          --commit-sha "${{ github.sha }}"
+          breeze prod-image build --builder airflow_cache --tag-as-latest
+          --image-tag "${{ inputs.image-tag }}" --commit-sha "${{ github.sha }}"
           --install-packages-from-context --airflow-constraints-mode constraints
           --use-constraints-for-context-packages --python "${{ matrix.python-version }}"
         env:

.github/workflows/push-image-cache.yml

@@ -74,8 +74,8 @@ on:  # yamllint disable-line rule:truthy
         required: true
         type: string
 jobs:
-  push-image-cache:
-    name: "Push CI ${{ inputs.cache-type }} image cache"
+  push-ci-image-cache:
+    name: "Push CI ${{ inputs.cache-type }}:${{ matrix.python }} image cache "
     runs-on: ${{ fromJSON(inputs.runs-on) }}
     strategy:
       fail-fast: false
@@ -120,12 +120,12 @@ jobs:
         if: always() && inputs.platform == 'linux/arm64'
       - name: "Push CI latest images: ${{ matrix.python }} (linux/amd64 only)"
         run: >
-          breeze ci-image build --push
+          breeze ci-image build --builder airflow_cache --push
           --python "${{ matrix.python }}" --platform "${{ inputs.platform }}"
         if: inputs.push-latest-images == 'true' && inputs.platform == 'linux/amd64'
 
   push-prod-image-cache:
-    name: "Push PROD ${{ inputs.cache-type }} image cache"
+    name: "Push PROD ${{ inputs.cache-type }}:${{ matrix.python }} image cache"
     runs-on: ${{ fromJSON(inputs.runs-on) }}
     strategy:
       fail-fast: false
@@ -186,6 +186,6 @@ jobs:
         # and we can do it after we stopped the ARM instance as it is not needed anymore
       - name: "Push PROD latest image: ${{ matrix.python }} (linux/amd64 ONLY)"
         run: >
-          breeze prod-image build --install-packages-from-context
-          --push --python ${{ matrix.python}} --platform "${{ inputs.platform }}"
+          breeze prod-image build --builder airflow_cache --install-packages-from-context
+          --push --platform "${{ inputs.platform }}"
         if: inputs.push-latest-images == 'true' && inputs.platform == 'linux/amd64'