Replace FAI with our own implementation

Just short of 700 new lines of Python, but covers running mmdebstrap and lots of workarounds go away. I've titled it "minifai". grml-policyrcd gets removed, as minifai handles installing its own policy-rc.d. This is necessary regardless if we want grml-policyrcd or not, and then grml-policyrcd provides no value anymore. The variables FAI_ARGS, FAI_DEBOOTSTRAP_OPTS and ZERO_FAI_LOGFILE are no longer supported. The grml-live parameter -V is now always enabled and the option is deprecated. minifai logs to stdout/stderr, and aborts on the first error it encounters. All log messages now end up in grml-live.log. These log/data files are no longer written: debconf.data, fai-monitor.log, fai.conf, fcopy.log, nfsroot.conf, scripts.log, software.log, status.log, task_error, task_local_error, variables.log. Notably missing: package "deinstall" support.
grml · Jan 6, 2025 · 2e9860d · 2e9860d
1 parent 268db08
commit 2e9860d
Show file tree

Hide file tree

Showing 13 changed files with 773 additions and 397 deletions.
diff --git a/build-driver/build.py b/build-driver/build.py
@@ -111,22 +111,6 @@ def run_grml_live(
         }
     )
 
-    if not old_iso_path:
-        with ci_section("Creating basefile using mmdebstrap"):
-            basefiles_path = grml_fai_config / "basefiles"
-            basefiles_path.mkdir(exist_ok=True)
-            basefile = basefiles_path / f"{arch.upper()}.tar.gz"
-            args = [
-                "mmdebstrap",
-                "--format=tar",
-                "--variant=required",
-                "--verbose",
-                "--include=netbase",
-                debian_suite,
-                basefile,
-            ]
-            run_x(args)
-
     grml_live_cmd = [
         grml_live_path / "grml-live",
         "-F",  # do not prompt
@@ -152,19 +136,9 @@ def run_grml_live(
     if old_iso_path:
         grml_live_cmd += ["-b", "-e", old_iso_path]
     with ci_section("Building with grml-live", collapsed=False):
-        fixup_fai()
         run_x(grml_live_cmd, env=env)
 
 
-def fixup_fai():
-    # Workaround for fai, necessary to build in docker where /dev/pts is unavailable.
-    # apt prints: E: Can not write log (Is /dev/pts mounted?) - posix_openpt (19: No such device)
-    fai_subroutines = Path("/usr/lib/fai/subroutines")
-    old_code = fai_subroutines.read_text().splitlines()
-    filtered_code = "\n".join([line for line in old_code if "task_error 472" not in line])
-    fai_subroutines.write_text(filtered_code)
-
-
 def upload_daily(job_name: str, build_dir: Path, job_timestamp: datetime.datetime):
     ssh_key = os.getenv("DAILY_UPLOAD_SSH_KEY")
     remote = os.getenv("DAILY_UPLOAD_REMOTE")
@@ -319,14 +293,14 @@ def install_debian_dependencies():
             bzip2 ,
             curl ,
             dosfstools ,
-            fai-client (>= 3.4.0) ,
             jo ,
             mmdebstrap ,
             moreutils ,
             mtools ,
             python3-paramiko ,
             rsync ,
             squashfs-tools (>= 1:4.2-0~bpo60) ,
+            socat ,
             xorriso ,
             imagemagick ,
             """

diff --git a/config/class/GRMLBASE.var b/config/class/GRMLBASE.var
@@ -3,17 +3,6 @@
 # allow installation of packages from unsigned repositories
 FAI_ALLOW_UNSIGNED=1
 
-CONSOLEFONT=
-KEYMAP=us-latin1
-
 # Set UTC=yes if your system clock is set to UTC (GMT) and UTC=no if not.
 UTC=no
 TIMEZONE=UTC
-
-# root password for the new installed linux system; md5 and crypt are possible
-ROOTPW='x'
-
-# Maximum number of packages installed at a time,
-# needs to be quite high so we can override installation
-# of specific packages through the IGNORE class.
-MAXPACKAGES=3000
diff --git a/config/hooks/instsoft.GRMLBASE b/config/hooks/instsoft.GRMLBASE
@@ -24,40 +24,35 @@ if [ -n "$BUILD_ONLY" ] ; then
    exit 0
 fi
 
-if [ "$FAI_ACTION" = "softupdate" ] ; then
-   echo "Action $FAI_ACTION of FAI (hooks/instsoft.GRMLBASE) via grml-live running"
+echo "hooks/instsoft.GRMLBASE running for action ${FAI_ACTION}"
+
+# work around /etc/kernel/postinst.d/zz-update-grub failing
+# inside openvz environment, see #597084
+if ! $ROOTCMD dpkg-divert --list | grep -q '/usr/sbin/update-grub' ; then
+  echo "Diverting update-grub executable"
+  $ROOTCMD dpkg-divert --rename --add /usr/sbin/update-grub
+  $ROOTCMD ln -s /bin/true /usr/sbin/update-grub
+fi
+
+# work around a bug which causes openvz to freeze when grub-probe is invoked
+if ! $ROOTCMD dpkg-divert --list | grep -q '/usr/sbin/grub-probe' ; then
+  echo "Diverting grub-probe executable"
+  $ROOTCMD dpkg-divert --rename --add /usr/sbin/grub-probe
+  $ROOTCMD ln -s /bin/true /usr/sbin/grub-probe
+fi
 
+# Do not fail running softupdate for reported bugs.
+[ -d "${target}"/etc/apt/apt.conf.d ] || mkdir "${target}"/etc/apt/apt.conf.d
+if [ -e "${target}"/etc/apt/apt.conf.d/10apt-listbugs ]; then
+    mv "${target}"/etc/apt/apt.conf.d/10apt-listbugs "${target}"/etc/apt/apt.conf.d/10apt-listbugs.disabled
+fi
+
+if [ "$FAI_ACTION" = "softupdate" ] ; then
    # /etc/resolv.conf is usually a symlink, pointing out of the chroot.
    # Make it a file with known contents.
    rm -f "${target}"/etc/resolv.conf
    cat /etc/resolv.conf >> "${target}"/etc/resolv.conf
 
-   if [ -r "${target}"/etc/policy-rc.d.conf ] ; then
-      sed -i "s/EXITSTATUS=.*/EXITSTATUS='101'/" "${target}"/etc/policy-rc.d.conf
-   fi
-
-   # we definitely don't want to fail running fai sofupdate just
-   # because of some well known bugs:
-   [ -d "${target}"/etc/apt/apt.conf.d ] || mkdir "${target}"/etc/apt/apt.conf.d
-   if [ -e "${target}"/etc/apt/apt.conf.d/10apt-listbugs ]; then
-       mv "${target}"/etc/apt/apt.conf.d/10apt-listbugs "${target}"/etc/apt/apt.conf.d/10apt-listbugs.disabled
-   fi
-
-   # work around /etc/kernel/postinst.d/zz-update-grub failing
-   # inside openvz environment, see #597084
-   if ! $ROOTCMD dpkg-divert --list | grep -q '/usr/sbin/update-grub' ; then
-     echo "Diverting update-grub executable"
-     $ROOTCMD dpkg-divert --rename --add /usr/sbin/update-grub
-     $ROOTCMD ln -s /bin/true /usr/sbin/update-grub
-   fi
-
-   # work around a bug which causes openvz to freeze when grub-probe is invoked
-   if ! $ROOTCMD dpkg-divert --list | grep -q '/usr/sbin/grub-probe' ; then
-     echo "Diverting grub-probe executable"
-     $ROOTCMD dpkg-divert --rename --add /usr/sbin/grub-probe
-     $ROOTCMD ln -s /bin/true /usr/sbin/grub-probe
-   fi
-
    # Update package lists (so they exist at all), so we can install
    # software; if /var/lib/dpkg/available is empty, it was was probably
    # cleaned by GRMLBASE/98-clean-chroot, so we need to rebuild it
@@ -100,44 +95,14 @@ if [ "$FAI_ACTION" = "softupdate" ] ; then
       APT_LISTCHANGES_FRONTEND=none APT_LISTBUGS_FRONTEND=none $ROOTCMD apt-get -y $APTGET_OPTS --force-yes upgrade
    fi
 
-   exit # make sure we don't continue behind the following "fi"
+   exit  # make sure we don't continue behind the following "fi"
 fi
 
-# no softupdate but fresh installation
-echo "Action ${FAI_ACTION} of FAI (hooks/instsoft.GRMLBASE) via grml-live running"
+# fresh installation, not softupdate.
 
-# work around /etc/kernel/postinst.d/zz-update-grub failing
-# inside openvz environment, see #597084
-if ! $ROOTCMD dpkg-divert --list | grep -q '/usr/sbin/update-grub' ; then
-  echo "Diverting update-grub executable"
-  $ROOTCMD dpkg-divert --rename --add /usr/sbin/update-grub
-  $ROOTCMD ln -s /bin/true /usr/sbin/update-grub
-fi
-
-# work around a bug which causes openvz to freeze when grub-probe is invoked
-if ! $ROOTCMD dpkg-divert --list | grep -q '/usr/sbin/grub-probe' ; then
-  echo "Diverting grub-probe executable"
-  $ROOTCMD dpkg-divert --rename --add /usr/sbin/grub-probe
-  $ROOTCMD ln -s /bin/true /usr/sbin/grub-probe
-fi
+# no hacks here, for now.
 
 # }}}
 
-# we definitely don't want to fail running fai dirinstall just
-# because of some well known bugs:
-[ -d "${target}"/etc/apt/apt.conf.d ] || mkdir "${target}"/etc/apt/apt.conf.d
-if [ -e "${target}"/etc/apt/apt.conf.d/10apt-listbugs ]; then
-    mv "${target}"/etc/apt/apt.conf.d/10apt-listbugs "${target}"/etc/apt/apt.conf.d/10apt-listbugs.disabled
-fi
-
-# we don't need the invoke-rc.d.d diversion (we have grml-policyrcd :)):
-if [ -L "${target}"/usr/sbin/invoke-rc.d ] ; then
-   rm -f "${target}"/usr/sbin/invoke-rc.d
-   $ROOTCMD dpkg-divert --package fai --rename --remove /usr/sbin/invoke-rc.d
-fi
-
-echo "[instsoft] Removing FAI diversion of /sbin/init to avoid Debian bug #1056151 for fai < 6.2.3"
-fai-divert -r /sbin/init || true
-
 ## END OF FILE #################################################################
 # vim:ft=sh expandtab ai tw=80 tabstop=4 shiftwidth=2
diff --git a/config/hooks/updatebase.GRMLBASE b/config/hooks/updatebase.GRMLBASE
@@ -32,31 +32,10 @@ if [ "$FAI_ACTION" = "softupdate" ] ; then
    # otherwise we're running 'aptitude update' even on with -b option
    skiptask updatebase
 
-   ## based on FAI's lib/updatebase:
-   # some packages must access /proc even in chroot environment
-   if ! [ -d "$FAI_ROOT"/proc/1 ] ; then
-      mount -t proc proc "$FAI_ROOT"/proc || true
-   fi
-   # some packages must access /sys even in chroot environment
-   if ! [ -d "$FAI_ROOT"/sys/kernel ] ; then
-      mount -t sysfs sysfs "$FAI_ROOT"/sys || true
-   fi
-   # if we are using udev, also mount it into $FAI_ROOT
-   if [ -f /etc/init.d/udev ] ; then
-      mount --bind /dev "$FAI_ROOT"/dev || true
-   fi
-
-   if [ -d "$FAI_ROOT"/run ] ; then
-      mount -t tmpfs tmpfs "$FAI_ROOT"/run || true
-      mkdir "$FAI_ROOT"/run/lock
-   fi
-
-   mount -t devpts devpts "$FAI_ROOT"/dev/pts || true
-
    # skip the task if we want to build a new ISO only,
    # this means we do NOT update any packages
    if [ -n "$BUILD_ONLY" ] ; then
-      skiptask instsoft || true
+      skiptask instsoft
    fi
 fi
 

diff --git a/config/package_config/GRMLBASE b/config/package_config/GRMLBASE
@@ -21,7 +21,6 @@ grml-etc
 grml-etc-core
 grml-hwinfo
 grml-network
-grml-policyrcd
 grml-quickconfig
 grml-scripts
 grml-scripts-core

diff --git a/config/scripts/GRMLBASE/01-packages b/config/scripts/GRMLBASE/01-packages
@@ -12,47 +12,31 @@ set -e
 # FAI sets $target, but shellcheck does not know that.
 target=${target:?}
 
+echo "Validating package list against dpkg state..."
 echo -n > "${LOGDIR}"/package_errors.log  # ensure we start with an empty file
 
-if ! [ -e "${LOGDIR}"/software.log ] ; then
-  echo "Warning: no ${LOGDIR}/software.log found (build/update run?), skipping check for unknown packages."
-else
-  if grep -q 'These unknown packages' "${LOGDIR}"/software.log ; then
-    echo "Identified unknown packages in ${LOGDIR}/software.log"
-    grep 'These unknown packages' "${LOGDIR}"/software.log | \
-      sed 's/.*These unknown packages.*: //; s/ / not_installable\n/g' >> "${LOGDIR}/package_errors.log"
-  fi
-fi
-
-PACKAGE_LIST=/var/log/install_packages.list
-# shellcheck disable=SC2154
-if ! [ -r "${target}/${PACKAGE_LIST}" ] ; then
-  echo "No ${target}/${PACKAGE_LIST} found, will not run package validation check."
-else
-  echo "Validating package list against dpkg state..."
-
-  TMPSTDOUT=$(mktemp)
-  TMPSTDERR=$(mktemp)
+TMPSTDOUT=$(mktemp)
+TMPSTDERR=$(mktemp)
 
-  # 1) catch packages that aren't in state 'ii' (marked for installation
-  # and successfully instead) on stdout
-  # 2) catch error messages like 'dpkg-query: no packages found matching $package"
-  # for packages unknown to dpkg on stderr
-  # NOTE: 'grep -v -- '-$' ignores packages in FAI's package list that are
-  # marked for removal
-  # shellcheck disable=SC2046
-  ${ROOTCMD} dpkg --list $(grep -v '^#' "${target}/${PACKAGE_LIST}" | grep -v -- '-$') 2>"${TMPSTDERR}" | \
-    grep -e '^[urph][ncufhWt]' > "${TMPSTDOUT}" || true
+PACKAGE_LIST=/var/log/grml-live-minifai/install_packages.list
+# 1) catch packages that are not in state 'ii' (marked for installation
+# and successfully instead) on stdout
+# 2) catch error messages like 'dpkg-query: no packages found matching $package"
+# for packages unknown to dpkg on stderr
+# NOTE: 'grep -v -- '-$' ignores packages in FAI's package list that are
+# marked for removal
+# shellcheck disable=SC2046
+${ROOTCMD} dpkg --list $(grep -v '^#' "${target}/${PACKAGE_LIST}" | grep -v -- '-$') 2>"${TMPSTDERR}" | \
+  grep -e '^[urph][ncufhWt]' > "${TMPSTDOUT}" || true
 
-  # extract packages from stdout
-  awk '/^un/ {print $2 " not_installable"}' "${TMPSTDOUT}" >> "${LOGDIR}/package_errors.log"
+# extract packages from stdout
+awk '/^un/ {print $2 " not_installable"}' "${TMPSTDOUT}" >> "${LOGDIR}/package_errors.log"
 
-  # extract packages from stderr
-  grep 'packages found matching' "${TMPSTDERR}" | \
-    sed 's/dpkg-query: [Nn]o packages found matching \(.*\)/\1 not_installable/' >> "${LOGDIR}/package_errors.log"
+# extract packages from stderr
+grep 'packages found matching' "${TMPSTDERR}" | \
+  sed 's/dpkg-query: [Nn]o packages found matching \(.*\)/\1 not_installable/' >> "${LOGDIR}/package_errors.log"
 
-  rm -f "${TMPSTDOUT}" "${TMPSTDERR}"
-fi
+rm -f "${TMPSTDOUT}" "${TMPSTDERR}"
 
 if [ -s "${LOGDIR}/package_errors.log" ] ; then
   echo "Warning: failed (there have been errors, find them at ${LOGDIR}/package_errors.log)."

diff --git a/config/scripts/GRMLBASE/98-clean-chroot b/config/scripts/GRMLBASE/98-clean-chroot
@@ -84,8 +84,6 @@ if [ -d "$target"/var/spool/squid/ ] ; then
 fi
 
 echo "Cleaning and removing some misc files and directories"
-find "$target"/etc -type f -name '*.pre_fcopy' -delete
-find "$target"/etc -type l -name '*.pre_fcopy' -delete
 rm -rf --one-file-system "$target"/etc/sysconfig/* \
        "$target"/etc/motd.dpkg-* "$target"/etc/auto.master.*dpkg* \
        "$target"/etc/samba/*.SID "$target"/etc/samba/*.tdb \
@@ -111,7 +109,7 @@ nuke(){
 zero(){
   while IFS= read -r -d '' file ; do
     :> "$file"
-  done < <(find "$@" -type f -size +0 -not -name \*.ini -not -path '*/fai/*' -not -name install_packages.list -print0 2>/dev/null)
+  done < <(find "$@" -type f -size +0 -not -name \*.ini -not -path '*/grml-live-minifai/*' -print0 2>/dev/null)
 }
 
 echo "Removing possible leftovers from update-pciids runs"

diff --git a/config/scripts/GRMLBASE/99-finish-grml-build b/config/scripts/GRMLBASE/99-finish-grml-build
@@ -12,11 +12,6 @@ set -e
 # FAI sets $target, but shellcheck does not know that.
 target=${target:?}
 
-# Restore original state from softupdate:
-if [ -r "$target"/etc/policy-rc.d.conf ] ; then
-   sed -i "s/EXITSTATUS='101'/EXITSTATUS='0'/" "$target"/etc/policy-rc.d.conf
-fi
-
 # remove an existing /etc/debian_chroot file:
 if [ -r "$target"/etc/debian_chroot ] ; then
    rm -f "$target"/etc/debian_chroot

diff --git a/debian/control b/debian/control
@@ -22,11 +22,11 @@ Depends:
  bzip2,
  debootstrap,
  dosfstools,
- fai-client (>= 3.4.0),
  jo,
  moreutils,
  mtools,
  rsync,
+ socat,
  squashfs-tools (>= 1:4.2-0~bpo60),
  xorriso,
  ${misc:Depends},