Add ARM image building for regular PRs

The image building for ARM is currently only done in the main build
only to refresh cache, however there are sometimes cases when
new dependency (for example #24635) broke ARM image build and it
was only discovered after merge.

This PR adds extra ARM-based build that should be run after
the AMD64 build. It should not influence the depending steps,
it should just signal failure of the PR if the ARM image cannot
be build.

.github/workflows/build-images.yml

@@ -225,21 +225,35 @@ jobs:
       - name: "Free space"
         run: breeze free-space
       - name: >
-          Build & Push CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+          Build & Push AMD64 CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
           ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
         run: breeze build-image --push-image --tag-as-latest --run-in-parallel
+        if: matrix.platform == 'linux/amd64'
+        env:
+          UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
+          DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+          IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+          PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+      - name: "Start ARM instance"
+        run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
+        if: matrix.platform == 'linux/arm64'
+      - name: >
+          Build ARM CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+          ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+        run: breeze build-image --run-in-parallel
+        if: matrix.platform == 'linux/arm64'
         env:
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
           DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
           IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
           PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
       - name: Push empty CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
-        if: failure() || cancelled()
+        if: (failure() || cancelled()) && matrix.platform == 'linux/amd64'
         run: breeze build-image --push-image --empty-image --run-in-parallel
         env:
           IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
       - name: "Candidates for pip resolver backtrack triggers"
-        if: failure() || cancelled()
+        if: (failure() || cancelled()) && matrix.platform == 'linux/amd64'
         run: >
           breeze find-newer-dependencies --max-age 1
           --python "${{ needs.build-info.outputs.defaultPythonVersion }}"
@@ -360,3 +374,86 @@ jobs:
       - name: "Fix ownership"
         run: breeze fix-ownership
         if: always()
+
+
+  build-ci-images-arm:
+    timeout-minutes: 80
+    name: "Build ARM CI images ${{ needs.build-info.outputs.allPythonVersionsListAsString }}"
+    runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+    needs: [build-info]
+    if: |
+      needs.build-info.outputs.image-build == 'true' &&
+      github.event.pull_request.head.repo.full_name != 'apache/airflow'
+    env:
+      RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
+      BACKEND: sqlite
+      outputs: ${{toJSON(needs.build-info.outputs) }}
+    steps:
+      - name: Cleanup repo
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ needs.build-info.outputs.targetCommitSha }}
+          persist-credentials: false
+          submodules: recursive
+      - name: "Retrieve DEFAULTS from the _initialization.sh"
+        # We cannot "source" the script here because that would be a security problem (we cannot run
+        # any code that comes from the sources coming from the PR. Therefore, we extract the
+        # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands
+        id: defaults
+        run: |
+          DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
+            awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+          echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
+          DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
+            scripts/ci/libraries/_initialization.sh | \
+            awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+          echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
+          DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \
+            awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+          echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
+      - name: >
+          Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to 'main-airflow' folder
+          to use ci/scripts from there.
+        uses: actions/checkout@v2
+        with:
+          path: "main-airflow"
+          ref: "${{ needs.build-info.outputs.targetBranch }}"
+          persist-credentials: false
+          submodules: recursive
+      - name: >
+          Override "scripts/ci" with the "${{ needs.build-info.outputs.targetBranch }}" branch
+          so that the PR does not override it
+        # We should not override those scripts which become part of the image as they will not be
+        # changed in the image built - we should only override those that are executed to build
+        # the image.
+        run: |
+          rm -rfv "scripts/ci"
+          rm -rfv "dev"
+          mv -v "main-airflow/scripts/ci" "scripts"
+          mv -v "main-airflow/dev" "."
+      - name: "Setup python"
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+      - run: ./scripts/ci/install_breeze.sh
+      - name: "Free space"
+        run: breeze free-space
+      - name: "Start ARM instance"
+        run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
+      - name: >
+          Build ARM CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+          ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+        run: breeze build-image --run-in-parallel --builder airflow_cache --platform "linux/arm64"
+        env:
+          UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
+          DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+          IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+          PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+        continue-on-error: true  # for now to make sure that it works in main
+      - name: "Stop ARM instance"
+        run: ./scripts/ci/images/ci_stop_arm_instance.sh
+        if: always()
+      - name: "Fix ownership"
+        run: breeze fix-ownership
+        if: always()

.github/workflows/ci.yml

@@ -351,6 +351,73 @@ jobs:
         run: breeze fix-ownership
         if: always() && needs.build-info.outputs.inWorkflowBuild == 'true'
 
+  build-ci-arm-images:
+    timeout-minutes: 80
+    name: >
+      ${{needs.build-info.outputs.buildJobDescription}} CI ARM images
+      ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+    runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+    needs: [build-info]
+    env:
+      RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
+    steps:
+      - name: Cleanup repo
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+        if: needs.build-info.outputs.inWorkflowBuild == 'true'
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ needs.build-info.outputs.targetCommitSha }}
+          persist-credentials: false
+          submodules: recursive
+        if: needs.build-info.outputs.inWorkflowBuild == 'true'
+      - name: "Setup python"
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+        if: needs.build-info.outputs.inWorkflowBuild == 'true'
+      - name: "Retrieve DEFAULTS from the _initialization.sh"
+        # We cannot "source" the script here because that would be a security problem (we cannot run
+        # any code that comes from the sources coming from the PR. Therefore, we extract the
+        # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands
+        id: defaults
+        run: |
+          DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
+            awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+          echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
+          DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
+            scripts/ci/libraries/_initialization.sh | \
+            awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+          echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
+          DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \
+            awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+          echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
+        if: needs.build-info.outputs.inWorkflowBuild == 'true'
+      - run: ./scripts/ci/install_breeze.sh
+        if: needs.build-info.outputs.inWorkflowBuild == 'true'
+      - name: "Free space"
+        run: breeze free-space
+        if: needs.build-info.outputs.inWorkflowBuild == 'true'
+      - name: "Start ARM instance"
+        run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
+        if: needs.build-info.outputs.inWorkflowBuild == 'true'
+      - name: >
+          Build CI ARM images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+          ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+        run: breeze build-image --run-in-parallel  --builder airflow_cache --platform "linux/arm64"
+        env:
+          UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
+          DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+          IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+          PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+        continue-on-error: true
+        if: needs.build-info.outputs.inWorkflowBuild == 'true'
+      - name: "Stop ARM instance"
+        run: ./scripts/ci/images/ci_stop_arm_instance.sh
+        if: always() && needs.build-info.outputs.inWorkflowBuild == 'true'
+      - name: "Fix ownership"
+        run: breeze fix-ownership
+        if: always() && needs.build-info.outputs.inWorkflowBuild == 'true'
+
   build-prod-images:
     permissions:
       packages: write

dev/breeze/src/airflow_breeze/commands/ci_image_commands.py

@@ -37,6 +37,7 @@
     option_airflow_constraints_mode_ci,
     option_airflow_constraints_reference_build,
     option_answer,
+    option_builder,
     option_debian_version,
     option_dev_apt_command,
     option_dev_apt_deps,
@@ -252,6 +253,7 @@ def run_build_in_parallel(
 @option_additional_dev_apt_env
 @option_additional_runtime_apt_env
 @option_additional_runtime_apt_command
+@option_builder
 @option_dev_apt_command
 @option_dev_apt_deps
 @option_force_build

dev/breeze/src/airflow_breeze/commands/production_image_commands.py

@@ -37,6 +37,7 @@
     option_airflow_constraints_mode_prod,
     option_airflow_constraints_reference_build,
     option_answer,
+    option_builder,
     option_debian_version,
     option_dev_apt_command,
     option_dev_apt_deps,
@@ -301,6 +302,7 @@ def run_build_in_parallel(
 @option_additional_dev_apt_env
 @option_additional_runtime_apt_env
 @option_additional_runtime_apt_command
+@option_builder
 @option_dev_apt_command
 @option_dev_apt_deps
 @option_python_image

dev/breeze/src/airflow_breeze/params/common_build_params.py

@@ -45,6 +45,7 @@ class CommonBuildParams:
     airflow_constraints_location: str = ""
     answer: Optional[str] = None
     build_id: int = 0
+    builder: str = "default"
     constraints_github_repository: str = "apache/airflow"
     debian_version: str = "bullseye"
     dev_apt_command: str = ""

dev/breeze/src/airflow_breeze/utils/common_options.py

@@ -441,10 +441,15 @@
     is_flag=True,
     envvar='PULL_IMAGE',
 )
-
 option_python_image = click.option(
     '--python-image',
     help="If specified this is the base python image used to build the image. "
     "Should be something like: python:VERSION-slim-bullseye",
     envvar='PYTHON_IMAGE',
 )
+option_builder = click.option(
+    '--builder',
+    help="Buildx builder used to perform `docker buildx build` commands",
+    envvar='BUILDER',
+    default='default',
+)

dev/breeze/src/airflow_breeze/utils/docker_command_utils.py

@@ -352,7 +352,7 @@ def prepare_docker_build_cache_command(
     build_flags = image_params.extra_docker_build_flags
     final_command = []
     final_command.extend(["docker"])
-    final_command.extend(["buildx", "build", "--builder", "airflow_cache", "--progress=tty"])
+    final_command.extend(["buildx", "build", "--builder", image_params.builder, "--progress=tty"])
     final_command.extend(build_flags)
     final_command.extend(["--pull"])
     final_command.extend(arguments)
@@ -388,7 +388,7 @@ def prepare_base_build_command(image_params: CommonBuildParams, verbose: bool) -
                 "buildx",
                 "build",
                 "--builder",
-                "default",
+                image_params.builder,
                 "--progress=tty",
                 "--push" if image_params.push_image else "--load",
             ]