feat: Build packages for Debian armhf (but no publish)

Use the new image names from `mender-test-containers` to build both the Raspberry Pi OS armhf packages (previously known as Debian armhf) and actual Debian armhf packages. See: * mendersoftware/mender-test-containers#628 Note that the publishing must be prevented until we re-design the APT repository structure. Ticket: MEN-7731 Signed-off-by: Lluis Campos <[email protected]>
mendersoftware · Nov 21, 2024 · d0f5bcb · d0f5bcb
1 parent a92138b
commit d0f5bcb
Show file tree

Hide file tree

Showing 4 changed files with 94 additions and 56 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -74,20 +74,22 @@ variables:
   AWS_EC2_METADATA_DISABLED: "true"
 
 # NOTE: To add distributions, modify first the matrix in mender-test-containers repository
-.mender-dist-packages-image-matrix:
+.mender-dist-packages-image-matrix-cross:
   parallel:
     matrix:
-      - DISTRO: debian
+      - LABEL: cross-debian
+        DISTRO: debian
         RELEASE: [bullseye, bookworm]
-        ARCH: [amd64, arm64]
-      - DISTRO: ubuntu
+        ARCH: [amd64, armhf, arm64]
+      - LABEL: cross-ubuntu
+        DISTRO: ubuntu
         RELEASE: [focal, jammy, noble]
         ARCH: [amd64, armhf, arm64]
-
-.mender-dist-packages-image-matrix-armhf:
+.mender-dist-packages-image-matrix-sim:
   parallel:
     matrix:
-      - DISTRO: debian
+      - LABEL: sim-raspberrypios
+        DISTRO: raspberrypios
         RELEASE: [bullseye, bookworm]
         ARCH: [armhf]
 
@@ -124,6 +126,7 @@ build:orig:debian:bullseye:amd64:
     - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:26-dind
       alias: docker
   variables:
+    LABEL: "cross-debian"
     DISTRO: "debian"
     RELEASE: "bullseye"
     ARCH: "amd64"
@@ -132,23 +135,23 @@ build:orig:debian:bullseye:amd64:
   script:
     - apk --update --no-cache add bash
     - if [[ "${MENDER_VERSION}" =~ ^3\..* ]]; then
-    -   ./docker-build-package $DISTRO $RELEASE $ARCH mender-client $MENDER_VERSION true
+    -   ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-client $MENDER_VERSION true
     - else
-    -   ./docker-build-package $DISTRO $RELEASE $ARCH mender-client4 $MENDER_VERSION true
+    -   ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-client4 $MENDER_VERSION true
     - fi
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-connect $MENDER_CONNECT_VERSION true
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-configure $MENDER_CONFIGURE_VERSION true
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-artifact $MENDER_ARTIFACT_VERSION true
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-app-update-module $MENDER_APP_UPDATE_MODULE_VERSION true
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-setup $MENDER_SETUP_VERSION true
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-snapshot $MENDER_SNAPSHOT_VERSION true
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-flash $MENDER_FLASH_VERSION true
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-connect $MENDER_CONNECT_VERSION true
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-configure $MENDER_CONFIGURE_VERSION true
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-artifact $MENDER_ARTIFACT_VERSION true
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-app-update-module $MENDER_APP_UPDATE_MODULE_VERSION true
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-setup $MENDER_SETUP_VERSION true
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-snapshot $MENDER_SNAPSHOT_VERSION true
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-flash $MENDER_FLASH_VERSION true
     - find output -type f
   artifacts:
     paths:
       - output/orig
 
-build:packages:
+build:packages:cross:
   stage: build:packages
   rules:
     - if: $CI_COMMIT_BRANCH == "production"
@@ -165,33 +168,33 @@ build:packages:
   script:
     - apk --update --no-cache add bash
     - if [[ "${MENDER_VERSION}" =~ ^3\..* ]]; then
-    -   ./docker-build-package $DISTRO $RELEASE $ARCH mender-client $MENDER_VERSION
+    -   ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-client $MENDER_VERSION
     - else
-    -   ./docker-build-package $DISTRO $RELEASE $ARCH mender-client4 $MENDER_VERSION
+    -   ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-client4 $MENDER_VERSION
     - fi
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-connect $MENDER_CONNECT_VERSION
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-configure $MENDER_CONFIGURE_VERSION
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-artifact $MENDER_ARTIFACT_VERSION
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-app-update-module $MENDER_APP_UPDATE_MODULE_VERSION
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-setup $MENDER_SETUP_VERSION
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-snapshot $MENDER_SNAPSHOT_VERSION
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-flash $MENDER_FLASH_VERSION
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-connect $MENDER_CONNECT_VERSION
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-configure $MENDER_CONFIGURE_VERSION
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-artifact $MENDER_ARTIFACT_VERSION
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-app-update-module $MENDER_APP_UPDATE_MODULE_VERSION
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-setup $MENDER_SETUP_VERSION
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-snapshot $MENDER_SNAPSHOT_VERSION
+    - ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-flash $MENDER_FLASH_VERSION
     - if [ -n "${MENDER_PRIVATE_REPO_ACCESS_TOKEN}" ]; then
-    -   ./docker-build-package $DISTRO $RELEASE $ARCH mender-gateway $MENDER_GATEWAY_VERSION
-    -   ./docker-build-package $DISTRO $RELEASE $ARCH mender-monitor $MENDER_MONITOR_VERSION
+    -   ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-gateway $MENDER_GATEWAY_VERSION
+    -   ./docker-build-package $LABEL $DISTRO $RELEASE $ARCH mender-monitor $MENDER_MONITOR_VERSION
     - fi
     - find output -type f
   artifacts:
     paths:
       - output/commercial
       - output/opensource
-  parallel: !reference [.mender-dist-packages-image-matrix, parallel]
+  parallel: !reference [.mender-dist-packages-image-matrix-cross, parallel]
 
-build:packages:armhf:
-  extends: build:packages
+build:packages:sim:
+  extends: build:packages:cross
   tags:
     - hetzner-arm
-  parallel: !reference [.mender-dist-packages-image-matrix-armhf, parallel]
+  parallel: !reference [.mender-dist-packages-image-matrix-sim, parallel]
 
 test:check-commits:
   only:
@@ -213,9 +216,9 @@ test:check-python3-formatting:
     - mender-qa-worker-generic
   needs:
     # Note that we are only testing packages from Debian bullseye
-    - "build:packages: [debian, bullseye, amd64]"
-    - "build:packages:armhf: [debian, bullseye, armhf]"
-    - "build:packages: [debian, bullseye, arm64]"
+    - "build:packages:cross: [cross-debian, debian, bullseye, amd64]"
+    - "build:packages:sim: [sim-raspberrypios, raspberrypios, bullseye, armhf]"
+    - "build:packages:cross: [cross-debian, debian, bullseye, arm64]"
   variables:
     GIT_SUBMODULE_STRATEGY: recursive
   before_script:
@@ -325,6 +328,28 @@ test:acceptance:script:
   }
 
 .template:publish:s3:apt-repo:
+  ################ MEN-7731 Debian armhf packages #####################
+  # We have support for building Debian armhf packages but the APT repositories are not ready
+  # so we cannot publish these packages. For the time being, we have explicitly here the dependencies
+  # so that the jobs: [cross-debian, debian, bullseye/bookworm, armhf]' are not pulled into the
+  # publish stage. Remove the whole dependencies section when working on MEN-7754
+  dependencies:
+    - 'build:orig:debian:bullseye:amd64'
+    - 'build:packages:cross: [cross-debian, debian, bullseye, amd64]'
+    - 'build:packages:cross: [cross-debian, debian, bullseye, arm64]'
+    - 'build:packages:cross: [cross-debian, debian, bookworm, amd64]'
+    - 'build:packages:cross: [cross-debian, debian, bookworm, arm64]'
+    - 'build:packages:cross: [cross-ubuntu, ubuntu, focal, amd64]'
+    - 'build:packages:cross: [cross-ubuntu, ubuntu, focal, armhf]'
+    - 'build:packages:cross: [cross-ubuntu, ubuntu, focal, arm64]'
+    - 'build:packages:cross: [cross-ubuntu, ubuntu, jammy, amd64]'
+    - 'build:packages:cross: [cross-ubuntu, ubuntu, jammy, armhf]'
+    - 'build:packages:cross: [cross-ubuntu, ubuntu, jammy, arm64]'
+    - 'build:packages:cross: [cross-ubuntu, ubuntu, noble, armhf]'
+    - 'build:packages:cross: [cross-ubuntu, ubuntu, noble, arm64]'
+    - 'build:packages:cross: [cross-ubuntu, ubuntu, noble, amd64]'
+    - 'build:packages:sim: [sim-raspberrypios, raspberrypios, bullseye, armhf]'
+    - 'build:packages:sim: [sim-raspberrypios, raspberrypios, bookworm, armhf]'
   stage: publish
   image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/debian:12
   tags:

diff --git a/docker-build-package b/docker-build-package
@@ -104,16 +104,20 @@ declare -a packages=(
 )
 
 # Parse args
-if [ $# -lt 4 ]; then
-    echo "usage: $0 distro release arch package [version]"
+if [ $# -lt 5 ]; then
+    echo "usage: $0 label distro release arch package [version]"
     exit 1
 fi
-DISTRO="${1}"
-RELEASE="${2}"
-ARCH="${3}"
-RECIPE_NAME="${4}"
-RECIPE_VERSION="${5:-master}"
-SAVE_ORIG="$6"
+IMAGE_LABEL="${1}"
+DISTRO="${2}"
+RELEASE="${3}"
+ARCH="${4}"
+RECIPE_NAME="${5}"
+RECIPE_VERSION="${6:-master}"
+SAVE_ORIG="$7"
+
+CONTAINER_TAG=mender-dist-packages-builder-${LABEL}-${RELEASE}-${ARCH}
+
 
 # Look for the recipe
 recipe_found=0
@@ -173,7 +177,7 @@ fi
 
 PLATFORM=""
 
-if [ "$ARCH" = "armhf" -a "$DISTRO" = "debian" ]; then
+if [ "$ARCH" = "armhf" -a "$DISTRO" = "raspberrypios" ]; then
     PLATFORM="--platform=linux/arm/v6"
 fi
 
@@ -188,7 +192,7 @@ docker run --rm \
         --env MENDER_PRIVATE_GPG_KEY_BUILD \
         --env OVERRIDE_DEBIAN_SUFFIX \
         ${PLATFORM} \
-        ${IMAGE_NAME_PREFIX}-${DISTRO}-${RELEASE}-${ARCH}-${IMAGE_VERSION:-master} \
+        ${IMAGE_NAME_PREFIX}-${IMAGE_LABEL}-${RELEASE}-${ARCH}-${IMAGE_VERSION:-master} \
         /script \
         ${recipe_name} \
         ${BUILD_TYPE} \

diff --git a/mender-deb-package b/mender-deb-package
@@ -63,7 +63,7 @@ install_go() {
   local GOLANG_VERSION=1.21.1
   local GOLANG_ARCH=amd64
   local golang_version_set
-  if [ $ARCH = "armhf" -a $OS_DISTRO = "debian" ]; then
+  if [ $ARCH = "armhf" -a $OS_DISTRO = "raspbian" ]; then
     local GOLANG_ARCH=armv6l
   fi
 
@@ -80,11 +80,6 @@ install_go() {
 
 get_os_version() {
   OS_DISTRO="$(. /etc/os-release && echo "$ID")"
-  # The images for armhf have the distro set to raspbian.
-  # Replace with debian to avoid changing all debian checks
-  if [ $OS_DISTRO = "raspbian" ]; then
-      OS_DISTRO="debian"
-  fi
   OS_CODENAME="$(. /etc/os-release && echo "$VERSION_CODENAME")"
 }
 
@@ -113,7 +108,14 @@ get_deb_version() {
   if [ -n "${OVERRIDE_DEBIAN_SUFFIX}" ]; then
     debian_suffix="${OVERRIDE_DEBIAN_SUFFIX}"
   fi
-  debian_suffix="$debian_suffix+$OS_DISTRO+$OS_CODENAME"
+  if [ $OS_DISTRO = "raspbian" ]; then
+    ################ MEN-7731 Debian armhf packages #####################
+    # For the raspbian builds, use the old "Debian armhf" for the time being. We need to redesign
+    # the APT repositories in order to separate these two armhf builds. Rework this with MEN-7754
+    debian_suffix="$debian_suffix+debian+$OS_CODENAME"
+  else
+    debian_suffix="$debian_suffix+$OS_DISTRO+$OS_CODENAME"
+  fi
   if [ "$VERSION" != "master" ] && git describe --tags --exact-match 2>/dev/null; then
     DEB_VERSION="$(git describe --tags --exact-match)"
     DEB_VERSION_NO_REV="${DEB_VERSION}"
@@ -240,7 +242,7 @@ build_packages() {
       ;;
 
     armhf)
-      if [ "$OS_DISTRO" = "debian" ]; then
+      if [ "$OS_DISTRO" = "raspbian" ]; then
         # Native build (emulated ARM v6)
         dpkg-buildpackage \
           ${sign_flags} \

diff --git a/tests/helpers.py b/tests/helpers.py
@@ -31,10 +31,17 @@
 def packages_path(package, package_arch="armhf"):
     if package_arch == "all":
         package_arch = "amd64"
-    subdir = "opensource/debian-bullseye-" + package_arch
+
+    distro = "debian"
+    if package_arch == "armhf":
+        # For armhf, we test only on Raspberry Pi OS, so it safe to assume distro here
+        distro = "raspberrypios"
+
+    basedir = "opensource"
     if package in COMMERCIAL_PACKAGES:
-        subdir = "commercial/debian-bullseye-" + package_arch
-    return os.path.join(output_path, subdir)
+        basedir = "commercial"
+
+    return os.path.join(output_path, basedir, f"{distro}-bullseye-{package_arch}")
 
 
 def package_filename(package_version, package_name, package_arch="armhf"):