Add ARM image building for regular PRs

.github/workflows/build-images.yml

@@ -225,21 +225,35 @@ jobs:
       - name: "Free space"
         run: breeze free-space
       - name: >
-          Build & Push CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+          Build & Push AMD64 CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
           ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
         run: breeze build-image --push-image --tag-as-latest --run-in-parallel
+        if: matrix.platform == 'linux/amd64'
+        env:
+          UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
+          DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+          IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+          PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+      - name: "Start ARM instance"
+        run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
+        if: matrix.platform == 'linux/arm64'
+      - name: >
+          Build ARM CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+          ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+        run: breeze build-image --run-in-parallel
+        if: matrix.platform == 'linux/arm64'
         env:
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
           DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
           IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
           PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
       - name: Push empty CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
-        if: failure() || cancelled()
+        if: (failure() || cancelled()) && matrix.platform == 'linux/amd64'
         run: breeze build-image --push-image --empty-image --run-in-parallel
         env:
           IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
       - name: "Candidates for pip resolver backtrack triggers"
-        if: failure() || cancelled()
+        if: (failure() || cancelled()) && matrix.platform == 'linux/amd64'
         run: >
           breeze find-newer-dependencies --max-age 1
           --python "${{ needs.build-info.outputs.defaultPythonVersion }}"
@@ -360,3 +374,86 @@ jobs:
       - name: "Fix ownership"
         run: breeze fix-ownership
         if: always()
+
+
+  build-ci-images-arm:
+    timeout-minutes: 80
+    name: "Build ARM CI images ${{ needs.build-info.outputs.allPythonVersionsListAsString }}"
+    runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+    needs: [build-info]
+    if: |
+      needs.build-info.outputs.image-build == 'true' &&
+      github.event.pull_request.head.repo.full_name != 'apache/airflow'
+    env:
+      RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
+      BACKEND: sqlite
+      outputs: ${{toJSON(needs.build-info.outputs) }}
+    steps:
+      - name: Cleanup repo
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ needs.build-info.outputs.targetCommitSha }}
+          persist-credentials: false
+          submodules: recursive
+      - name: "Retrieve DEFAULTS from the _initialization.sh"
+        # We cannot "source" the script here because that would be a security problem (we cannot run
+        # any code that comes from the sources coming from the PR. Therefore, we extract the
+        # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands
+        id: defaults
+        run: |
+          DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
+            awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+          echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
+          DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
+            scripts/ci/libraries/_initialization.sh | \
+            awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+          echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
+          DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \
+            awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+          echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
+      - name: >
+          Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to 'main-airflow' folder
+          to use ci/scripts from there.
+        uses: actions/checkout@v2
+        with:
+          path: "main-airflow"
+          ref: "${{ needs.build-info.outputs.targetBranch }}"
+          persist-credentials: false
+          submodules: recursive
+      - name: >
+          Override "scripts/ci" with the "${{ needs.build-info.outputs.targetBranch }}" branch
+          so that the PR does not override it
+        # We should not override those scripts which become part of the image as they will not be
+        # changed in the image built - we should only override those that are executed to build
+        # the image.
+        run: |
+          rm -rfv "scripts/ci"
+          rm -rfv "dev"
+          mv -v "main-airflow/scripts/ci" "scripts"
+          mv -v "main-airflow/dev" "."
+      - name: "Setup python"
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+      - run: ./scripts/ci/install_breeze.sh
+      - name: "Free space"
+        run: breeze free-space
+      - name: "Start ARM instance"
+        run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
+      - name: >
+          Build ARM CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+          ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+        run: breeze build-image --run-in-parallel --builder airflow_cache --platform "linux/arm64"
+        env:
+          UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
+          DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+          IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+          PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+        continue-on-error: true  # for now to make sure that it works in main
+      - name: "Stop ARM instance"
+        run: ./scripts/ci/images/ci_stop_arm_instance.sh
+        if: always()
+      - name: "Fix ownership"
+        run: breeze fix-ownership
+        if: always()

.github/workflows/ci.yml

@@ -351,6 +351,73 @@ jobs:
         run: breeze fix-ownership
         if: always() && needs.build-info.outputs.inWorkflowBuild == 'true'
 
+  build-ci-arm-images:
+    timeout-minutes: 80
+    name: >
+      ${{needs.build-info.outputs.buildJobDescription}} CI ARM images
+      ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+    runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+    needs: [build-info]
+    env:
+      RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
+    steps:
+      - name: Cleanup repo
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+        if: needs.build-info.outputs.inWorkflowBuild == 'true'
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ needs.build-info.outputs.targetCommitSha }}
+          persist-credentials: false
+          submodules: recursive
+        if: needs.build-info.outputs.inWorkflowBuild == 'true'
+      - name: "Setup python"
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+        if: needs.build-info.outputs.inWorkflowBuild == 'true'
+      - name: "Retrieve DEFAULTS from the _initialization.sh"
+        # We cannot "source" the script here because that would be a security problem (we cannot run
+        # any code that comes from the sources coming from the PR. Therefore, we extract the
+        # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands
+        id: defaults
+        run: |
+          DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
+            awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+          echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
+          DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
+            scripts/ci/libraries/_initialization.sh | \
+            awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+          echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
+          DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \
+            awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+          echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
+        if: needs.build-info.outputs.inWorkflowBuild == 'true'
+      - run: ./scripts/ci/install_breeze.sh
+        if: needs.build-info.outputs.inWorkflowBuild == 'true'
+      - name: "Free space"
+        run: breeze free-space
+        if: needs.build-info.outputs.inWorkflowBuild == 'true'
+      - name: "Start ARM instance"
+        run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
+        if: needs.build-info.outputs.inWorkflowBuild == 'true'
+      - name: >
+          Build CI ARM images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+          ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+        run: breeze build-image --run-in-parallel  --builder airflow_cache --platform "linux/arm64"
+        env:
+          UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
+          DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+          IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+          PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+        continue-on-error: true
+        if: needs.build-info.outputs.inWorkflowBuild == 'true'
+      - name: "Stop ARM instance"
+        run: ./scripts/ci/images/ci_stop_arm_instance.sh
+        if: always() && needs.build-info.outputs.inWorkflowBuild == 'true'
+      - name: "Fix ownership"
+        run: breeze fix-ownership
+        if: always() && needs.build-info.outputs.inWorkflowBuild == 'true'
+
   build-prod-images:
     permissions:
       packages: write

dev/breeze/src/airflow_breeze/commands/ci_image_commands.py

@@ -37,6 +37,7 @@
     option_airflow_constraints_mode_ci,
     option_airflow_constraints_reference_build,
     option_answer,
+    option_builder,
     option_debian_version,
     option_dev_apt_command,
     option_dev_apt_deps,
@@ -252,6 +253,7 @@ def run_build_in_parallel(
 @option_additional_dev_apt_env
 @option_additional_runtime_apt_env
 @option_additional_runtime_apt_command
+@option_builder
 @option_dev_apt_command
 @option_dev_apt_deps
 @option_force_build

dev/breeze/src/airflow_breeze/commands/production_image_commands.py

@@ -37,6 +37,7 @@
     option_airflow_constraints_mode_prod,
     option_airflow_constraints_reference_build,
     option_answer,
+    option_builder,
     option_debian_version,
     option_dev_apt_command,
     option_dev_apt_deps,
@@ -301,6 +302,7 @@ def run_build_in_parallel(
 @option_additional_dev_apt_env
 @option_additional_runtime_apt_env
 @option_additional_runtime_apt_command
+@option_builder
 @option_dev_apt_command
 @option_dev_apt_deps
 @option_python_image

dev/breeze/src/airflow_breeze/params/common_build_params.py

@@ -45,6 +45,7 @@ class CommonBuildParams:
     airflow_constraints_location: str = ""
     answer: Optional[str] = None
     build_id: int = 0
+    builder: str = "default"
     constraints_github_repository: str = "apache/airflow"
     debian_version: str = "bullseye"
     dev_apt_command: str = ""

dev/breeze/src/airflow_breeze/utils/common_options.py

@@ -441,10 +441,15 @@
     is_flag=True,
     envvar='PULL_IMAGE',
 )
-
 option_python_image = click.option(
     '--python-image',
     help="If specified this is the base python image used to build the image. "
     "Should be something like: python:VERSION-slim-bullseye",
     envvar='PYTHON_IMAGE',
 )
+option_builder = click.option(
+    '--builder',
+    help="Buildx builder used to perform `docker buildx build` commands",
+    envvar='BUILDER',
+    default='default',
+)

dev/breeze/src/airflow_breeze/utils/docker_command_utils.py

@@ -352,7 +352,7 @@ def prepare_docker_build_cache_command(
     build_flags = image_params.extra_docker_build_flags
     final_command = []
     final_command.extend(["docker"])
-    final_command.extend(["buildx", "build", "--builder", "airflow_cache", "--progress=tty"])
+    final_command.extend(["buildx", "build", "--builder", image_params.builder, "--progress=tty"])
     final_command.extend(build_flags)
     final_command.extend(["--pull"])
     final_command.extend(arguments)
@@ -388,7 +388,7 @@ def prepare_base_build_command(image_params: CommonBuildParams, verbose: bool) -
                 "buildx",
                 "build",
                 "--builder",
-                "default",
+                image_params.builder,
                 "--progress=tty",
                 "--push" if image_params.push_image else "--load",
             ]