feat: Build packages for Debian armhf (but no publish)

Use the new image names from `mender-test-containers` to build both the Raspberry Pi OS armhf packages (previously known as Debian armhf) and actual Debian armhf packages. Note that the publishing must be prevented until we re-design the APT repository structure. Ticket: MEN-7731 Signed-off-by: Lluis Campos <[email protected]>
mendersoftware · Nov 20, 2024 · 2624415 · 2624415
1 parent d42125c
commit 2624415
Show file tree

Hide file tree

Showing 2 changed files with 74 additions and 46 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -72,20 +72,22 @@ variables:
   AWS_EC2_METADATA_DISABLED: "true"
 
 # NOTE: To add distributions, modify first the matrix in mender-test-containers repository
-.mender-dist-packages-image-matrix:
+.mender-dist-packages-image-matrix-cross:
   parallel:
     matrix:
-      - DISTRO: debian
+      - LABEL: cross-debian
+        DISTRO: debian
         RELEASE: [bullseye, bookworm]
-        ARCH: [amd64, arm64]
-      - DISTRO: ubuntu
+        ARCH: [amd64, armhf, arm64]
+      - LABEL: cross-ubuntu
+        DISTRO: ubuntu
         RELEASE: [focal, jammy, noble]
         ARCH: [amd64, armhf, arm64]
-
-.mender-dist-packages-image-matrix-armhf:
+.mender-dist-packages-image-matrix-sim:
   parallel:
     matrix:
-      - DISTRO: debian
+      - LABEL: sim-raspberrypios
+        DISTRO: raspberrypios
         RELEASE: [bullseye, bookworm]
         ARCH: [armhf]
 
@@ -130,23 +132,23 @@ build:orig:debian:bullseye:amd64:
   script:
     - apk --update --no-cache add bash
     - if [[ "${MENDER_VERSION}" =~ ^3\..* ]]; then
-    -   ./docker-build-package $DISTRO $RELEASE $ARCH mender-client $MENDER_VERSION true
+    -   ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-client $MENDER_VERSION true
     - else
-    -   ./docker-build-package $DISTRO $RELEASE $ARCH mender-client4 $MENDER_VERSION true
+    -   ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-client4 $MENDER_VERSION true
     - fi
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-connect $MENDER_CONNECT_VERSION true
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-configure $MENDER_CONFIGURE_VERSION true
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-artifact $MENDER_ARTIFACT_VERSION true
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-app-update-module $MENDER_APP_UPDATE_MODULE_VERSION true
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-setup $MENDER_SETUP_VERSION true
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-snapshot $MENDER_SNAPSHOT_VERSION true
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-flash $MENDER_FLASH_VERSION true
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-connect $MENDER_CONNECT_VERSION true
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-configure $MENDER_CONFIGURE_VERSION true
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-artifact $MENDER_ARTIFACT_VERSION true
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-app-update-module $MENDER_APP_UPDATE_MODULE_VERSION true
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-setup $MENDER_SETUP_VERSION true
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-snapshot $MENDER_SNAPSHOT_VERSION true
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-flash $MENDER_FLASH_VERSION true
     - find output -type f
   artifacts:
     paths:
       - output/orig
 
-build:packages:
+build:packages:cross:
   stage: build:packages
   rules:
     - if: $CI_COMMIT_BRANCH == "production"
@@ -163,33 +165,33 @@ build:packages:
   script:
     - apk --update --no-cache add bash
     - if [[ "${MENDER_VERSION}" =~ ^3\..* ]]; then
-    -   ./docker-build-package $DISTRO $RELEASE $ARCH mender-client $MENDER_VERSION
+    -   ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-client $MENDER_VERSION
     - else
-    -   ./docker-build-package $DISTRO $RELEASE $ARCH mender-client4 $MENDER_VERSION
+    -   ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-client4 $MENDER_VERSION
     - fi
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-connect $MENDER_CONNECT_VERSION
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-configure $MENDER_CONFIGURE_VERSION
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-artifact $MENDER_ARTIFACT_VERSION
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-app-update-module $MENDER_APP_UPDATE_MODULE_VERSION
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-setup $MENDER_SETUP_VERSION
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-snapshot $MENDER_SNAPSHOT_VERSION
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-flash $MENDER_FLASH_VERSION
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-connect $MENDER_CONNECT_VERSION
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-configure $MENDER_CONFIGURE_VERSION
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-artifact $MENDER_ARTIFACT_VERSION
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-app-update-module $MENDER_APP_UPDATE_MODULE_VERSION
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-setup $MENDER_SETUP_VERSION
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-snapshot $MENDER_SNAPSHOT_VERSION
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-flash $MENDER_FLASH_VERSION
     - if [ -n "${MENDER_PRIVATE_REPO_ACCESS_TOKEN}" ]; then
-    -   ./docker-build-package $DISTRO $RELEASE $ARCH mender-gateway $MENDER_GATEWAY_VERSION
-    -   ./docker-build-package $DISTRO $RELEASE $ARCH mender-monitor $MENDER_MONITOR_VERSION
+    -   ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-gateway $MENDER_GATEWAY_VERSION
+    -   ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-monitor $MENDER_MONITOR_VERSION
     - fi
     - find output -type f
   artifacts:
     paths:
       - output/commercial
       - output/opensource
-  parallel: !reference [.mender-dist-packages-image-matrix, parallel]
+  parallel: !reference [.mender-dist-packages-image-matrix-cross, parallel]
 
-build:packages:armhf:
-  extends: build:packages
+build:packages:sim:
+  extends: build:packages:cross
   tags:
     - hetzner-arm
-  parallel: !reference [.mender-dist-packages-image-matrix-armhf, parallel]
+  parallel: !reference [.mender-dist-packages-image-matrix-sim, parallel]
 
 test:check-commits:
   only:
@@ -211,9 +213,9 @@ test:check-python3-formatting:
     - mender-qa-worker-generic
   needs:
     # Note that we are only testing packages from Debian bullseye
-    - "build:packages: [debian, bullseye, amd64]"
-    - "build:packages:armhf: [debian, bullseye, armhf]"
-    - "build:packages: [debian, bullseye, arm64]"
+    - "build:packages:cross: [cross-debian, debian, bullseye, amd64]"
+    - "build:packages:sim: [sim-raspberrypios, raspberrypios, bullseye, armhf]"
+    - "build:packages:cross: [cross-debian, debian, bullseye, arm64]"
   variables:
     GIT_SUBMODULE_STRATEGY: recursive
   before_script:
@@ -244,7 +246,7 @@ test:check-python3-formatting:
     - pip3 install --break-system-packages -r tests/requirements.txt
   script:
     - cd tests
-    - REFERENCE_DIST=debian-bullseye-amd64
+    - REFERENCE_DIST=cross-debian-bullseye-amd64
     - |-
         commercial_tests_flags=""
         if [ -f ${CI_PROJECT_DIR}/output/commercial/${REFERENCE_DIST}/mender-gateway-deb-version ] &&
@@ -323,6 +325,28 @@ test:acceptance:script:
   }
 
 .template:publish:s3:apt-repo:
+  ################ MEN-7731 Debian armhf packages #####################
+  # We have support for building Debian armhf packages but the APT repositories are not ready
+  # so we cannot publish these packages. For the time being, we have explicitly here the dependencies
+  # so that the jobs: [cross-debian, debian, bullseye/bookworm, armhf]' are not pulled into the
+  # publish stage. Remove the whole dependencies section when working on MEN-7754
+  dependencies:
+    - 'build:orig:debian:bullseye:amd64'
+    - 'build:packages:cross: [cross-debian, debian, bullseye, amd64]'
+    - 'build:packages:cross: [cross-debian, debian, bullseye, arm64]'
+    - 'build:packages:cross: [cross-debian, debian, bookworm, amd64]'
+    - 'build:packages:cross: [cross-debian, debian, bookworm, arm64]'
+    - 'build:packages:cross: [cross-ubuntu, ubuntu, focal, amd64]'
+    - 'build:packages:cross: [cross-ubuntu, ubuntu, focal, armhf]'
+    - 'build:packages:cross: [cross-ubuntu, ubuntu, focal, arm64]'
+    - 'build:packages:cross: [cross-ubuntu, ubuntu, jammy, amd64]'
+    - 'build:packages:cross: [cross-ubuntu, ubuntu, jammy, armhf]'
+    - 'build:packages:cross: [cross-ubuntu, ubuntu, jammy, arm64]'
+    - 'build:packages:cross: [cross-ubuntu, ubuntu, noble, armhf]'
+    - 'build:packages:cross: [cross-ubuntu, ubuntu, noble, arm64]'
+    - 'build:packages:cross: [cross-ubuntu, ubuntu, noble, amd64]'
+    - 'build:packages:sim: [sim-raspberrypios, raspberrypios, bullseye, armhf]'
+    - 'build:packages:sim: [sim-raspberrypios, raspberrypios, bookworm, armhf]'
   stage: publish
   image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/debian:12
   tags:

diff --git a/docker-build-package b/docker-build-package
@@ -104,16 +104,20 @@ declare -a packages=(
 )
 
 # Parse args
-if [ $# -lt 4 ]; then
-    echo "usage: $0 distro release arch package [version]"
+if [ $# -lt 5 ]; then
+    echo "usage: $0 label distro release arch package [version]"
     exit 1
 fi
-DISTRO="${1}"
-RELEASE="${2}"
-ARCH="${3}"
-RECIPE_NAME="${4}"
-RECIPE_VERSION="${5:-master}"
-SAVE_ORIG="$6"
+IMAGE_LABEL="${1}"
+DISTRO="${2}"
+RELEASE="${3}"
+ARCH="${4}"
+RECIPE_NAME="${5}"
+RECIPE_VERSION="${6:-master}"
+SAVE_ORIG="$7"
+
+CONTAINER_TAG=mender-dist-packages-builder-${LABEL}-${RELEASE}-${ARCH}
+
 
 # Look for the recipe
 recipe_found=0
@@ -173,7 +177,7 @@ fi
 
 PLATFORM=""
 
-if [ "$ARCH" = "armhf" -a "$DISTRO" = "debian" ]; then
+if [ "$ARCH" = "armhf" -a "$DISTRO" = "raspberrypios" ]; then
     PLATFORM="--platform=linux/arm/v6"
 fi
 
@@ -188,7 +192,7 @@ docker run --rm \
         --env MENDER_PRIVATE_GPG_KEY_BUILD \
         --env OVERRIDE_DEBIAN_SUFFIX \
         ${PLATFORM} \
-        ${IMAGE_NAME_PREFIX}-${DISTRO}-${RELEASE}-${ARCH}-${IMAGE_VERSION:-master} \
+        ${IMAGE_NAME_PREFIX}-${IMAGE_LABEL}-${RELEASE}-${ARCH}-${IMAGE_VERSION:-master} \
         /script \
         ${recipe_name} \
         ${BUILD_TYPE} \