MEN-7731: Build packages for Debian armhf (but no publish)

.gitlab-ci.yml

@@ -61,7 +61,9 @@ variables:
   # Workaround for GitLab intermittent networking
   FF_NETWORK_PER_BUILD: 1
   # Builder image version
-  IMAGE_VERSION: "master"
+  IMAGE_VERSION:
+    value: "master"
+    description: "Builder images version. Useful in combination with PRs from mender-test-containers"
   # Debian suffix
   OVERRIDE_DEBIAN_SUFFIX:
     value: ""
@@ -72,20 +74,22 @@ variables:
   AWS_EC2_METADATA_DISABLED: "true"
 
 # NOTE: To add distributions, modify first the matrix in mender-test-containers repository
-.mender-dist-packages-image-matrix:
+.mender-dist-packages-image-matrix-cross:
   parallel:
     matrix:
-      - DISTRO: debian
+      - BUILD: crosscompile
+        DISTRO: debian
         RELEASE: [bullseye, bookworm]
-        ARCH: [amd64, arm64]
-      - DISTRO: ubuntu
+        ARCH: [amd64, armhf, arm64]
+      - BUILD: crosscompile
+        DISTRO: ubuntu
         RELEASE: [focal, jammy, noble]
         ARCH: [amd64, armhf, arm64]
-
-.mender-dist-packages-image-matrix-armhf:
+.mender-dist-packages-image-matrix-sim:
   parallel:
     matrix:
-      - DISTRO: debian
+      - BUILD: simulated
+        DISTRO: raspberrypios
         RELEASE: [bullseye, bookworm]
         ARCH: [armhf]
 
@@ -122,6 +126,7 @@ build:orig:debian:bullseye:amd64:
     - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:26-dind
       alias: docker
   variables:
+    BUILD: "crosscompile"
     DISTRO: "debian"
     RELEASE: "bullseye"
     ARCH: "amd64"
@@ -130,23 +135,23 @@ build:orig:debian:bullseye:amd64:
   script:
     - apk --update --no-cache add bash
     - if [[ "${MENDER_VERSION}" =~ ^3\..* ]]; then
-    -   ./docker-build-package $DISTRO $RELEASE $ARCH mender-client $MENDER_VERSION true
+    -   ./docker-build-package $BUILD $DISTRO $RELEASE $ARCH mender-client $MENDER_VERSION true
     - else
-    -   ./docker-build-package $DISTRO $RELEASE $ARCH mender-client4 $MENDER_VERSION true
+    -   ./docker-build-package $BUILD $DISTRO $RELEASE $ARCH mender-client4 $MENDER_VERSION true
     - fi
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-connect $MENDER_CONNECT_VERSION true
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-configure $MENDER_CONFIGURE_VERSION true
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-artifact $MENDER_ARTIFACT_VERSION true
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-app-update-module $MENDER_APP_UPDATE_MODULE_VERSION true
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-setup $MENDER_SETUP_VERSION true
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-snapshot $MENDER_SNAPSHOT_VERSION true
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-flash $MENDER_FLASH_VERSION true
+    - ./docker-build-package $BUILD $DISTRO $RELEASE $ARCH mender-connect $MENDER_CONNECT_VERSION true
+    - ./docker-build-package $BUILD $DISTRO $RELEASE $ARCH mender-configure $MENDER_CONFIGURE_VERSION true
+    - ./docker-build-package $BUILD $DISTRO $RELEASE $ARCH mender-artifact $MENDER_ARTIFACT_VERSION true
+    - ./docker-build-package $BUILD $DISTRO $RELEASE $ARCH mender-app-update-module $MENDER_APP_UPDATE_MODULE_VERSION true
+    - ./docker-build-package $BUILD $DISTRO $RELEASE $ARCH mender-setup $MENDER_SETUP_VERSION true
+    - ./docker-build-package $BUILD $DISTRO $RELEASE $ARCH mender-snapshot $MENDER_SNAPSHOT_VERSION true
+    - ./docker-build-package $BUILD $DISTRO $RELEASE $ARCH mender-flash $MENDER_FLASH_VERSION true
     - find output -type f
   artifacts:
     paths:
       - output/orig
 
-build:packages:
+build:packages:cross:
   stage: build:packages
   rules:
     - if: $CI_COMMIT_BRANCH == "production"
@@ -163,33 +168,33 @@ build:packages:
   script:
     - apk --update --no-cache add bash
     - if [[ "${MENDER_VERSION}" =~ ^3\..* ]]; then
-    -   ./docker-build-package $DISTRO $RELEASE $ARCH mender-client $MENDER_VERSION
+    -   ./docker-build-package $BUILD $DISTRO $RELEASE $ARCH mender-client $MENDER_VERSION
     - else
-    -   ./docker-build-package $DISTRO $RELEASE $ARCH mender-client4 $MENDER_VERSION
+    -   ./docker-build-package $BUILD $DISTRO $RELEASE $ARCH mender-client4 $MENDER_VERSION
     - fi
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-connect $MENDER_CONNECT_VERSION
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-configure $MENDER_CONFIGURE_VERSION
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-artifact $MENDER_ARTIFACT_VERSION
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-app-update-module $MENDER_APP_UPDATE_MODULE_VERSION
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-setup $MENDER_SETUP_VERSION
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-snapshot $MENDER_SNAPSHOT_VERSION
-    - ./docker-build-package $DISTRO $RELEASE $ARCH mender-flash $MENDER_FLASH_VERSION
+    - ./docker-build-package $BUILD $DISTRO $RELEASE $ARCH mender-connect $MENDER_CONNECT_VERSION
+    - ./docker-build-package $BUILD $DISTRO $RELEASE $ARCH mender-configure $MENDER_CONFIGURE_VERSION
+    - ./docker-build-package $BUILD $DISTRO $RELEASE $ARCH mender-artifact $MENDER_ARTIFACT_VERSION
+    - ./docker-build-package $BUILD $DISTRO $RELEASE $ARCH mender-app-update-module $MENDER_APP_UPDATE_MODULE_VERSION
+    - ./docker-build-package $BUILD $DISTRO $RELEASE $ARCH mender-setup $MENDER_SETUP_VERSION
+    - ./docker-build-package $BUILD $DISTRO $RELEASE $ARCH mender-snapshot $MENDER_SNAPSHOT_VERSION
+    - ./docker-build-package $BUILD $DISTRO $RELEASE $ARCH mender-flash $MENDER_FLASH_VERSION
     - if [ -n "${MENDER_PRIVATE_REPO_ACCESS_TOKEN}" ]; then
-    -   ./docker-build-package $DISTRO $RELEASE $ARCH mender-gateway $MENDER_GATEWAY_VERSION
-    -   ./docker-build-package $DISTRO $RELEASE $ARCH mender-monitor $MENDER_MONITOR_VERSION
+    -   ./docker-build-package $BUILD $DISTRO $RELEASE $ARCH mender-gateway $MENDER_GATEWAY_VERSION
+    -   ./docker-build-package $BUILD $DISTRO $RELEASE $ARCH mender-monitor $MENDER_MONITOR_VERSION
     - fi
     - find output -type f
   artifacts:
     paths:
       - output/commercial
       - output/opensource
-  parallel: !reference [.mender-dist-packages-image-matrix, parallel]
+  parallel: !reference [.mender-dist-packages-image-matrix-cross, parallel]
 
-build:packages:armhf:
-  extends: build:packages
+build:packages:sim:
+  extends: build:packages:cross
   tags:
     - hetzner-arm
-  parallel: !reference [.mender-dist-packages-image-matrix-armhf, parallel]
+  parallel: !reference [.mender-dist-packages-image-matrix-sim, parallel]
 
 test:check-commits:
   only:
@@ -211,9 +216,9 @@ test:check-python3-formatting:
     - mender-qa-worker-generic
   needs:
     # Note that we are only testing packages from Debian bullseye
-    - "build:packages: [debian, bullseye, amd64]"
-    - "build:packages:armhf: [debian, bullseye, armhf]"
-    - "build:packages: [debian, bullseye, arm64]"
+    - "build:packages:cross: [crosscompile, debian, bullseye, amd64]"
+    - "build:packages:sim: [simulated, raspberrypios, bullseye, armhf]"
+    - "build:packages:cross: [crosscompile, debian, bullseye, arm64]"
   variables:
     GIT_SUBMODULE_STRATEGY: recursive
   before_script:
@@ -323,6 +328,28 @@ test:acceptance:script:
   }
 
 .template:publish:s3:apt-repo:
+  ################ MEN-7731 Debian armhf packages #####################
+  # We have support for building Debian armhf packages but the APT repositories are not ready
+  # so we cannot publish these packages. For the time being, we have explicitly here the dependencies
+  # so that the jobs: [crosscompile, debian, bullseye/bookworm, armhf]' are not pulled into the
+  # publish stage. Remove the whole dependencies section when working on MEN-7754
+  dependencies:
+    - 'build:orig:debian:bullseye:amd64'
+    - 'build:packages:cross: [crosscompile, debian, bullseye, amd64]'
+    - 'build:packages:cross: [crosscompile, debian, bullseye, arm64]'
+    - 'build:packages:cross: [crosscompile, debian, bookworm, amd64]'
+    - 'build:packages:cross: [crosscompile, debian, bookworm, arm64]'
+    - 'build:packages:cross: [crosscompile, ubuntu, focal, amd64]'
+    - 'build:packages:cross: [crosscompile, ubuntu, focal, armhf]'
+    - 'build:packages:cross: [crosscompile, ubuntu, focal, arm64]'
+    - 'build:packages:cross: [crosscompile, ubuntu, jammy, amd64]'
+    - 'build:packages:cross: [crosscompile, ubuntu, jammy, armhf]'
+    - 'build:packages:cross: [crosscompile, ubuntu, jammy, arm64]'
+    - 'build:packages:cross: [crosscompile, ubuntu, noble, armhf]'
+    - 'build:packages:cross: [crosscompile, ubuntu, noble, arm64]'
+    - 'build:packages:cross: [crosscompile, ubuntu, noble, amd64]'
+    - 'build:packages:sim: [simulated, raspberrypios, bullseye, armhf]'
+    - 'build:packages:sim: [simulated, raspberrypios, bookworm, armhf]'
   stage: publish
   image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/debian:12
   tags:

docker-build-package

@@ -104,16 +104,17 @@ declare -a packages=(
 )
 
 # Parse args
-if [ $# -lt 4 ]; then
-    echo "usage: $0 distro release arch package [version]"
+if [ $# -lt 5 ]; then
+    echo "usage: $0 build distro release arch package [version]"
     exit 1
 fi
-DISTRO="${1}"
-RELEASE="${2}"
-ARCH="${3}"
-RECIPE_NAME="${4}"
-RECIPE_VERSION="${5:-master}"
-SAVE_ORIG="$6"
+BUILD="${1}"
+DISTRO="${2}"
+RELEASE="${3}"
+ARCH="${4}"
+RECIPE_NAME="${5}"
+RECIPE_VERSION="${6:-master}"
+SAVE_ORIG="$7"
 
 # Look for the recipe
 recipe_found=0
@@ -173,7 +174,7 @@ fi
 
 PLATFORM=""
 
-if [ "$ARCH" = "armhf" -a "$DISTRO" = "debian" ]; then
+if [ "$ARCH" = "armhf" -a "$DISTRO" = "raspberrypios" ]; then
     PLATFORM="--platform=linux/arm/v6"
 fi
 
@@ -188,7 +189,7 @@ docker run --rm \
         --env MENDER_PRIVATE_GPG_KEY_BUILD \
         --env OVERRIDE_DEBIAN_SUFFIX \
         ${PLATFORM} \
-        ${IMAGE_NAME_PREFIX}-${DISTRO}-${RELEASE}-${ARCH}-${IMAGE_VERSION:-master} \
+        ${IMAGE_NAME_PREFIX}-${BUILD}-${DISTRO}-${RELEASE}-${ARCH}-${IMAGE_VERSION:-master} \
         /script \
         ${recipe_name} \
         ${BUILD_TYPE} \

mender-deb-package

@@ -63,7 +63,7 @@ install_go() {
   local GOLANG_VERSION=1.21.1
   local GOLANG_ARCH=amd64
   local golang_version_set
-  if [ $ARCH = "armhf" -a $OS_DISTRO = "debian" ]; then
+  if [ $ARCH = "armhf" -a $OS_DISTRO = "raspbian" ]; then
     local GOLANG_ARCH=armv6l
   fi
 
@@ -80,11 +80,6 @@ install_go() {
 
 get_os_version() {
   OS_DISTRO="$(. /etc/os-release && echo "$ID")"
-  # The images for armhf have the distro set to raspbian.
-  # Replace with debian to avoid changing all debian checks
-  if [ $OS_DISTRO = "raspbian" ]; then
-      OS_DISTRO="debian"
-  fi
   OS_CODENAME="$(. /etc/os-release && echo "$VERSION_CODENAME")"
 }
 
@@ -113,7 +108,14 @@ get_deb_version() {
   if [ -n "${OVERRIDE_DEBIAN_SUFFIX}" ]; then
     debian_suffix="${OVERRIDE_DEBIAN_SUFFIX}"
   fi
-  debian_suffix="$debian_suffix+$OS_DISTRO+$OS_CODENAME"
+  if [ $OS_DISTRO = "raspbian" ]; then
+    ################ MEN-7731 Debian armhf packages #####################
+    # For the raspbian builds, use the old "Debian armhf" for the time being. We need to redesign
+    # the APT repositories in order to separate these two armhf builds. Rework this with MEN-7754
+    debian_suffix="$debian_suffix+debian+$OS_CODENAME"
+  else
+    debian_suffix="$debian_suffix+$OS_DISTRO+$OS_CODENAME"
+  fi
   if [ "$VERSION" != "master" ] && git describe --tags --exact-match 2>/dev/null; then
     DEB_VERSION="$(git describe --tags --exact-match)"
     DEB_VERSION_NO_REV="${DEB_VERSION}"
@@ -240,7 +242,7 @@ build_packages() {
       ;;
 
     armhf)
-      if [ "$OS_DISTRO" = "debian" ]; then
+      if [ "$OS_DISTRO" = "raspbian" ]; then
         # Native build (emulated ARM v6)
         dpkg-buildpackage \
           ${sign_flags} \

tests/helpers.py

@@ -31,10 +31,17 @@
 def packages_path(package, package_arch="armhf"):
     if package_arch == "all":
         package_arch = "amd64"
-    subdir = "opensource/debian-bullseye-" + package_arch
+
+    distro = "debian"
+    if package_arch == "armhf":
+        # For armhf, we test only on Raspberry Pi OS, so it safe to assume distro here
+        distro = "raspberrypios"
+
+    basedir = "opensource"
     if package in COMMERCIAL_PACKAGES:
-        subdir = "commercial/debian-bullseye-" + package_arch
-    return os.path.join(output_path, subdir)
+        basedir = "commercial"
+
+    return os.path.join(output_path, basedir, f"{distro}-bullseye-{package_arch}")
 
 
 def package_filename(package_version, package_name, package_arch="armhf"):