From 6f96e58cb2b62955a9f1b4755ec635f780c52f49 Mon Sep 17 00:00:00 2001 From: Chris Hofstaedtler Date: Mon, 6 Jan 2025 02:18:44 +0100 Subject: [PATCH] Replace FAI with our own implementation Just short of 700 new lines of Python, but covers running mmdebstrap and lots of workarounds go away. grml-policyrcd gets removed, as minifai handles installing its own policy-rc.d. This is necessary regardless if we want grml-policyrcd or not, and then grml-policyrcd provides no value anymore. --- build-driver/build.py | 28 +- config/hooks/instsoft.GRMLBASE | 87 +-- config/hooks/updatebase.GRMLBASE | 23 +- config/package_config/GRMLBASE | 1 - config/scripts/GRMLBASE/01-packages | 53 +- config/scripts/GRMLBASE/99-finish-grml-build | 5 - debian/control | 1 - docs/grml-live.txt | 127 ++-- etc/grml/grml-live.conf | 6 - grml-live | 173 ++--- usr/lib/grml-live/minifai | 635 +++++++++++++++++++ 11 files changed, 766 insertions(+), 373 deletions(-) create mode 100755 usr/lib/grml-live/minifai diff --git a/build-driver/build.py b/build-driver/build.py index b134065c..fd6a0e51 100755 --- a/build-driver/build.py +++ b/build-driver/build.py @@ -111,22 +111,6 @@ def run_grml_live( } ) - if not old_iso_path: - with ci_section("Creating basefile using mmdebstrap"): - basefiles_path = grml_fai_config / "basefiles" - basefiles_path.mkdir(exist_ok=True) - basefile = basefiles_path / f"{arch.upper()}.tar.gz" - args = [ - "mmdebstrap", - "--format=tar", - "--variant=required", - "--verbose", - "--include=netbase", - debian_suite, - basefile, - ] - run_x(args) - grml_live_cmd = [ grml_live_path / "grml-live", "-F", # do not prompt @@ -152,19 +136,9 @@ def run_grml_live( if old_iso_path: grml_live_cmd += ["-b", "-e", old_iso_path] with ci_section("Building with grml-live", collapsed=False): - fixup_fai() run_x(grml_live_cmd, env=env) -def fixup_fai(): - # Workaround for fai, necessary to build in docker where /dev/pts is unavailable. - # apt prints: E: Can not write log (Is /dev/pts mounted?) - posix_openpt (19: No such device) - fai_subroutines = Path("/usr/lib/fai/subroutines") - old_code = fai_subroutines.read_text().splitlines() - filtered_code = "\n".join([line for line in old_code if "task_error 472" not in line]) - fai_subroutines.write_text(filtered_code) - - def upload_daily(job_name: str, build_dir: Path, job_timestamp: datetime.datetime): ssh_key = os.getenv("DAILY_UPLOAD_SSH_KEY") remote = os.getenv("DAILY_UPLOAD_REMOTE") @@ -319,7 +293,6 @@ def install_debian_dependencies(): bzip2 , curl , dosfstools , - fai-client (>= 3.4.0) , jo , mmdebstrap , moreutils , @@ -327,6 +300,7 @@ def install_debian_dependencies(): python3-paramiko , rsync , squashfs-tools (>= 1:4.2-0~bpo60) , + socat , xorriso , imagemagick , """ diff --git a/config/hooks/instsoft.GRMLBASE b/config/hooks/instsoft.GRMLBASE index b28748aa..c74778bf 100755 --- a/config/hooks/instsoft.GRMLBASE +++ b/config/hooks/instsoft.GRMLBASE @@ -24,40 +24,35 @@ if [ -n "$BUILD_ONLY" ] ; then exit 0 fi -if [ "$FAI_ACTION" = "softupdate" ] ; then - echo "Action $FAI_ACTION of FAI (hooks/instsoft.GRMLBASE) via grml-live running" +echo "hooks/instsoft.GRMLBASE running for action ${FAI_ACTION}" + +# work around /etc/kernel/postinst.d/zz-update-grub failing +# inside openvz environment, see #597084 +if ! $ROOTCMD dpkg-divert --list | grep -q '/usr/sbin/update-grub' ; then + echo "Diverting update-grub executable" + $ROOTCMD dpkg-divert --rename --add /usr/sbin/update-grub + $ROOTCMD ln -s /bin/true /usr/sbin/update-grub +fi + +# work around a bug which causes openvz to freeze when grub-probe is invoked +if ! $ROOTCMD dpkg-divert --list | grep -q '/usr/sbin/grub-probe' ; then + echo "Diverting grub-probe executable" + $ROOTCMD dpkg-divert --rename --add /usr/sbin/grub-probe + $ROOTCMD ln -s /bin/true /usr/sbin/grub-probe +fi +# Do not fail running softupdate for reported bugs. +[ -d "${target}"/etc/apt/apt.conf.d ] || mkdir "${target}"/etc/apt/apt.conf.d +if [ -e "${target}"/etc/apt/apt.conf.d/10apt-listbugs ]; then + mv "${target}"/etc/apt/apt.conf.d/10apt-listbugs "${target}"/etc/apt/apt.conf.d/10apt-listbugs.disabled +fi + +if [ "$FAI_ACTION" = "softupdate" ] ; then # /etc/resolv.conf is usually a symlink, pointing out of the chroot. # Make it a file with known contents. rm -f "${target}"/etc/resolv.conf cat /etc/resolv.conf >> "${target}"/etc/resolv.conf - if [ -r "${target}"/etc/policy-rc.d.conf ] ; then - sed -i "s/EXITSTATUS=.*/EXITSTATUS='101'/" "${target}"/etc/policy-rc.d.conf - fi - - # we definitely don't want to fail running fai sofupdate just - # because of some well known bugs: - [ -d "${target}"/etc/apt/apt.conf.d ] || mkdir "${target}"/etc/apt/apt.conf.d - if [ -e "${target}"/etc/apt/apt.conf.d/10apt-listbugs ]; then - mv "${target}"/etc/apt/apt.conf.d/10apt-listbugs "${target}"/etc/apt/apt.conf.d/10apt-listbugs.disabled - fi - - # work around /etc/kernel/postinst.d/zz-update-grub failing - # inside openvz environment, see #597084 - if ! $ROOTCMD dpkg-divert --list | grep -q '/usr/sbin/update-grub' ; then - echo "Diverting update-grub executable" - $ROOTCMD dpkg-divert --rename --add /usr/sbin/update-grub - $ROOTCMD ln -s /bin/true /usr/sbin/update-grub - fi - - # work around a bug which causes openvz to freeze when grub-probe is invoked - if ! $ROOTCMD dpkg-divert --list | grep -q '/usr/sbin/grub-probe' ; then - echo "Diverting grub-probe executable" - $ROOTCMD dpkg-divert --rename --add /usr/sbin/grub-probe - $ROOTCMD ln -s /bin/true /usr/sbin/grub-probe - fi - # Update package lists (so they exist at all), so we can install # software; if /var/lib/dpkg/available is empty, it was was probably # cleaned by GRMLBASE/98-clean-chroot, so we need to rebuild it @@ -100,44 +95,14 @@ if [ "$FAI_ACTION" = "softupdate" ] ; then APT_LISTCHANGES_FRONTEND=none APT_LISTBUGS_FRONTEND=none $ROOTCMD apt-get -y $APTGET_OPTS --force-yes upgrade fi - exit # make sure we don't continue behind the following "fi" + exit # make sure we don't continue behind the following "fi" fi -# no softupdate but fresh installation -echo "Action ${FAI_ACTION} of FAI (hooks/instsoft.GRMLBASE) via grml-live running" +# fresh installation, not softupdate. -# work around /etc/kernel/postinst.d/zz-update-grub failing -# inside openvz environment, see #597084 -if ! $ROOTCMD dpkg-divert --list | grep -q '/usr/sbin/update-grub' ; then - echo "Diverting update-grub executable" - $ROOTCMD dpkg-divert --rename --add /usr/sbin/update-grub - $ROOTCMD ln -s /bin/true /usr/sbin/update-grub -fi - -# work around a bug which causes openvz to freeze when grub-probe is invoked -if ! $ROOTCMD dpkg-divert --list | grep -q '/usr/sbin/grub-probe' ; then - echo "Diverting grub-probe executable" - $ROOTCMD dpkg-divert --rename --add /usr/sbin/grub-probe - $ROOTCMD ln -s /bin/true /usr/sbin/grub-probe -fi +# no hacks here, for now. # }}} -# we definitely don't want to fail running fai dirinstall just -# because of some well known bugs: -[ -d "${target}"/etc/apt/apt.conf.d ] || mkdir "${target}"/etc/apt/apt.conf.d -if [ -e "${target}"/etc/apt/apt.conf.d/10apt-listbugs ]; then - mv "${target}"/etc/apt/apt.conf.d/10apt-listbugs "${target}"/etc/apt/apt.conf.d/10apt-listbugs.disabled -fi - -# we don't need the invoke-rc.d.d diversion (we have grml-policyrcd :)): -if [ -L "${target}"/usr/sbin/invoke-rc.d ] ; then - rm -f "${target}"/usr/sbin/invoke-rc.d - $ROOTCMD dpkg-divert --package fai --rename --remove /usr/sbin/invoke-rc.d -fi - -echo "[instsoft] Removing FAI diversion of /sbin/init to avoid Debian bug #1056151 for fai < 6.2.3" -fai-divert -r /sbin/init || true - ## END OF FILE ################################################################# # vim:ft=sh expandtab ai tw=80 tabstop=4 shiftwidth=2 diff --git a/config/hooks/updatebase.GRMLBASE b/config/hooks/updatebase.GRMLBASE index b10c158f..a70b826e 100755 --- a/config/hooks/updatebase.GRMLBASE +++ b/config/hooks/updatebase.GRMLBASE @@ -32,31 +32,10 @@ if [ "$FAI_ACTION" = "softupdate" ] ; then # otherwise we're running 'aptitude update' even on with -b option skiptask updatebase - ## based on FAI's lib/updatebase: - # some packages must access /proc even in chroot environment - if ! [ -d "$FAI_ROOT"/proc/1 ] ; then - mount -t proc proc "$FAI_ROOT"/proc || true - fi - # some packages must access /sys even in chroot environment - if ! [ -d "$FAI_ROOT"/sys/kernel ] ; then - mount -t sysfs sysfs "$FAI_ROOT"/sys || true - fi - # if we are using udev, also mount it into $FAI_ROOT - if [ -f /etc/init.d/udev ] ; then - mount --bind /dev "$FAI_ROOT"/dev || true - fi - - if [ -d "$FAI_ROOT"/run ] ; then - mount -t tmpfs tmpfs "$FAI_ROOT"/run || true - mkdir "$FAI_ROOT"/run/lock - fi - - mount -t devpts devpts "$FAI_ROOT"/dev/pts || true - # skip the task if we want to build a new ISO only, # this means we do NOT update any packages if [ -n "$BUILD_ONLY" ] ; then - skiptask instsoft || true + skiptask instsoft fi fi diff --git a/config/package_config/GRMLBASE b/config/package_config/GRMLBASE index 5b9c746d..08897380 100644 --- a/config/package_config/GRMLBASE +++ b/config/package_config/GRMLBASE @@ -21,7 +21,6 @@ grml-etc grml-etc-core grml-hwinfo grml-network -grml-policyrcd grml-quickconfig grml-scripts grml-scripts-core diff --git a/config/scripts/GRMLBASE/01-packages b/config/scripts/GRMLBASE/01-packages index 181eaef7..29e88049 100755 --- a/config/scripts/GRMLBASE/01-packages +++ b/config/scripts/GRMLBASE/01-packages @@ -12,47 +12,30 @@ set -e # FAI sets $target, but shellcheck does not know that. target=${target:?} +echo "Validating package list against dpkg state..." echo -n > "${LOGDIR}"/package_errors.log # ensure we start with an empty file -if ! [ -e "${LOGDIR}"/software.log ] ; then - echo "Warning: no ${LOGDIR}/software.log found (build/update run?), skipping check for unknown packages." -else - if grep -q 'These unknown packages' "${LOGDIR}"/software.log ; then - echo "Identified unknown packages in ${LOGDIR}/software.log" - grep 'These unknown packages' "${LOGDIR}"/software.log | \ - sed 's/.*These unknown packages.*: //; s/ / not_installable\n/g' >> "${LOGDIR}/package_errors.log" - fi -fi - -PACKAGE_LIST=/var/log/install_packages.list -# shellcheck disable=SC2154 -if ! [ -r "${target}/${PACKAGE_LIST}" ] ; then - echo "No ${target}/${PACKAGE_LIST} found, will not run package validation check." -else - echo "Validating package list against dpkg state..." - - TMPSTDOUT=$(mktemp) - TMPSTDERR=$(mktemp) +TMPSTDOUT=$(mktemp) +TMPSTDERR=$(mktemp) - # 1) catch packages that aren't in state 'ii' (marked for installation - # and successfully instead) on stdout - # 2) catch error messages like 'dpkg-query: no packages found matching $package" - # for packages unknown to dpkg on stderr - # NOTE: 'grep -v -- '-$' ignores packages in FAI's package list that are - # marked for removal - # shellcheck disable=SC2046 - ${ROOTCMD} dpkg --list $(grep -v '^#' "${target}/${PACKAGE_LIST}" | grep -v -- '-$') 2>"${TMPSTDERR}" | \ - grep -e '^[urph][ncufhWt]' > "${TMPSTDOUT}" || true +# 1) catch packages that aren't in state 'ii' (marked for installation +# and successfully instead) on stdout +# 2) catch error messages like 'dpkg-query: no packages found matching $package" +# for packages unknown to dpkg on stderr +# NOTE: 'grep -v -- '-$' ignores packages in FAI's package list that are +# marked for removal +# shellcheck disable=SC2046 +${ROOTCMD} dpkg --list $(grep -v '^#' "${target}/${PACKAGE_LIST}" | grep -v -- '-$') 2>"${TMPSTDERR}" | \ + grep -e '^[urph][ncufhWt]' > "${TMPSTDOUT}" || true - # extract packages from stdout - awk '/^un/ {print $2 " not_installable"}' "${TMPSTDOUT}" >> "${LOGDIR}/package_errors.log" +# extract packages from stdout +awk '/^un/ {print $2 " not_installable"}' "${TMPSTDOUT}" >> "${LOGDIR}/package_errors.log" - # extract packages from stderr - grep 'packages found matching' "${TMPSTDERR}" | \ - sed 's/dpkg-query: [Nn]o packages found matching \(.*\)/\1 not_installable/' >> "${LOGDIR}/package_errors.log" +# extract packages from stderr +grep 'packages found matching' "${TMPSTDERR}" | \ + sed 's/dpkg-query: [Nn]o packages found matching \(.*\)/\1 not_installable/' >> "${LOGDIR}/package_errors.log" - rm -f "${TMPSTDOUT}" "${TMPSTDERR}" -fi +rm -f "${TMPSTDOUT}" "${TMPSTDERR}" if [ -s "${LOGDIR}/package_errors.log" ] ; then echo "Warning: failed (there have been errors, find them at ${LOGDIR}/package_errors.log)." diff --git a/config/scripts/GRMLBASE/99-finish-grml-build b/config/scripts/GRMLBASE/99-finish-grml-build index 827f2a0b..6da397ef 100755 --- a/config/scripts/GRMLBASE/99-finish-grml-build +++ b/config/scripts/GRMLBASE/99-finish-grml-build @@ -12,11 +12,6 @@ set -e # FAI sets $target, but shellcheck does not know that. target=${target:?} -# Restore original state from softupdate: -if [ -r "$target"/etc/policy-rc.d.conf ] ; then - sed -i "s/EXITSTATUS='101'/EXITSTATUS='0'/" "$target"/etc/policy-rc.d.conf -fi - # remove an existing /etc/debian_chroot file: if [ -r "$target"/etc/debian_chroot ] ; then rm -f "$target"/etc/debian_chroot diff --git a/debian/control b/debian/control index d30f3c51..ebdc4dab 100644 --- a/debian/control +++ b/debian/control @@ -22,7 +22,6 @@ Depends: bzip2, debootstrap, dosfstools, - fai-client (>= 3.4.0), jo, moreutils, mtools, diff --git a/docs/grml-live.txt b/docs/grml-live.txt index 583b6e02..593cbb11 100644 --- a/docs/grml-live.txt +++ b/docs/grml-live.txt @@ -4,8 +4,7 @@ grml-live(8) Name ---- -grml-live - build framework based on FAI for generating a Grml and Debian based -Linux Live system (CD/ISO) +grml-live - build framework for the Grml Live Linux system Synopsis -------- @@ -19,24 +18,20 @@ grml-live [-a ] [-c ] [-C ] [ Description ----------- -grml-live provides the build system for creating a Grml and Debian based Linux -Live-CD. The build system is based on -link:http://fai-project.org/[FAI] (Fully Automatic -Installation). grml-live uses the "fai dirinstall" feature to generate a chroot -system based on the class concept of FAI (see later sections for further -details) and provides the framework to be able to generate a full-featured ISO. -It does not use all the FAI features by default though and you don't have to -know FAI to be able to use it. - -The use of FAI gives you the flexibility to choose the packages you would like -to include on your very own Linux Live-CD without having to deal with all the -details of a build process. - -CAUTION: grml-live does **not** use /etc/fai for configuration but instead -provides and uses ${GRML_FAI_CONFIG} which is /usr/share/grml-live/config by default -(unless overridden using the ''-D'' option). This ensures that it does not clash -with default FAI configuration and packages, so you can use grml-live and FAI -completely independent at the same time! +grml-live is the build system for the Grml Live Linux system. You can also +use it to adapt a Grml release to your own needs. + +The build system is based on the class concept of link:http://fai-project.org/[FAI] +(Fully Automatic Installation). While older versions directly used FAI, the +current version of grml-live supports the same class-based configuration concept +but provides its own implementation. + +The class-based configuration concept gives you the flexibility to choose the +packages you would like to include on your own Linux Live-CD without having to +deal with all the details of a build process. + +CAUTION: grml-live no longer uses FAI. This means you are free to use FAI +separately in any way you like, but also you no longer need to have FAI installed. [NOTE] @@ -67,7 +62,7 @@ amd64 and arm64. -b:: -Build the ISO without updating the chroot via FAI. This option is useful for +Build the ISO without updating the chroot. This option is useful for example when working on stable releases: if you have a working base system/chroot and do not want to execute any further updates (via "-u" option) but intend to only build the ISO. @@ -75,13 +70,13 @@ but intend to only build the ISO. -B:: Build the ISO without touching the chroot at all. This option is useful if -you modified anything that FAI or grml-live might adjust via Grml's FAI -scripts. It's like the '-b' option but even more advanced. Use only if you -really know that you do not want to update the chroot. +you modified anything grml-live might adjust via the class-based configuration. +It's like the '-b' option but even more advanced. Use only if you really know +that you do not want to update the chroot. -c **CLASSES**:: -Specify the CLASSES to be used for building the ISO via FAI. By default only +Specify the CLASSES to be used for building the ISO. By default only the classes GRMLBASE, GRML_FULL and I386/AMD64/ARM64 (depending on system architecture) are assumed. Additionally you can specify a class providing a (grml-)kernel (see <> for @@ -122,13 +117,14 @@ advance. Usage example: '-d 2009-10-30' -D **CONFIGURATION_DIRECTORY**:: -The specified directory is used as configuration directory for grml-live and FAI. +The specified directory is used as configuration directory for grml-live. By default /usr/share/grml-live/config is used as default configuration directory. If you want to have different configuration scripts, package definitions, etc. without messing with the global configuration under /usr/share/grml-live/config provided by grml-live this option provides you the option to use your own configuration directory. -This directory is what's being referred to as ${GRML_FAI_CONFIG} throughout this documentation. +This directory is what's being referred to as ${GRML_FAI_CONFIG} throughout this +documentation. -e **EXTRACT_ISO_NAME**:: @@ -179,7 +175,7 @@ remaining stages and finalize the ISO. -o **OUTPUT_DIRECTORY**:: -Main output directory of the build process of FAI. Some directories are created +Main output directory of the build process. Some directories are created inside this target directory, being: grml_cd (where the files for creating the ISO are located, including the compressed squashfs file), grml_chroot (the chroot system) and grml_isos (where the resulting ISO is stored). @@ -214,8 +210,7 @@ Specify place of the templates used for building the ISO. By default -u:: -Update existing chroot instead of rebuilding it from scratch. This option is -based on the softupdate feature of FAI. +Update existing chroot instead of rebuilding it from scratch.. -U **USERNAME**:: @@ -284,15 +279,13 @@ Main features of grml-live * supports use and integration of own Software and/or Kernels via simple use of Debian repositories -* native support of FAI features - [[class-concept]] The class concept ----------------- -grml-live uses FAI and its class based concept for adjusting configuration and -setup according to your needs. This gives you flexibility and strength without -losing the simplicity in the build process. +grml-live uses a class based concept, like FAI, for adjusting configuration and +customizing the installation according to your needs. This gives you flexibility +and strength without losing the simplicity in the build process. The main and base class provided by grml-live is named GRMLBASE. It's strongly recommended to **always** use the class GRMLBASE when building an ISO using @@ -318,7 +311,9 @@ your amd64 build, et CLASSES="GRMLBASE,GRML_SMALL,AMD64,FOOBAR" inside "grml-live -c GRMLBASE,GRML_SMALL,AMD64,FOOBAR ...". More details regarding the class concept can be found in the documentation of -FAI itself (being available at /usr/share/doc/fai-doc/). +FAI (being available at /usr/share/doc/fai-doc/ if you install fai-doc). +In the past, grml-live directly used FAI. Nowadays, it uses an internal +implementation of the class concept. [[X7]] [[classes]] @@ -391,14 +386,13 @@ setup Files ----- -Notice that grml-live ships FAI configuration files that do not use the same -namespace as the FAI packages itself. This ensures that grml-live does not clash -with your usual FAI configuration. -For more details see below. To get an idea how another configuration or example -files could look like check out /usr/share/doc/fai-doc/examples/simple/ -(provided by Debian package fai-doc). Furthermore +To understand the class-based configuration concept, please refer to the +FAI documentation (provided by the Debian package fai-doc). Examples can +be found in /usr/share/doc/fai-doc/examples/simple/ . Furthermore /usr/share/doc/fai-doc/fai-guide.html/ch-config.html provides documentation -regarding configuration possibilities. +regarding configuration possibilities. Note that grml-live uses its own +implementation of the class-based concept, so not all FAI features are +available. /usr/sbin/grml-live @@ -420,12 +414,12 @@ commandline. ${GRML_FAI_CONFIG}/ -The main directory for configuration of FAI/grml-live. More details below. +The main directory for configuration of grml-live. More details below. ${GRML_FAI_CONFIG}/class/ This directory contains files which specify main configuration variables for the -FAI classes. +classes. ${GRML_FAI_CONFIG}/debconf/ @@ -565,9 +559,6 @@ Instructions CLASSES="GRMLBASE,GRML_FULL,AMD64" EOF - apt-get update - apt-get install fai-client fai-server fai-doc - That's it. Now invoking 'grml-live -V' should build the ISO. If everything worked as expected the last line of the shell output should look like: @@ -599,11 +590,8 @@ If you need help with grml-live or would like to see new features as part of grml-live you can get commercial support via link:http://grml-solutions.com/[Grml Solutions]. -Note that FAI doesn't abort immediately on errors that will ultimately cause -the build to fail. Be sure to look through the logs and find the actual error; -look for lines that start with "E: " or contain "FAILED" or "exit code 1". -Some messages that may look like errors are actually benign; e.g. -"/tmp/grml64/grml_chroot/var/lib/dpkg is not a mounted ramdisk" is not an error. +If there were errors during the build, grml-live should have aborted when +the error happened. [[install-local-files]] How do I install further files into the chroot/ISO? @@ -629,9 +617,9 @@ Can I use my own (local) Debian mirror? Yes. Set up an according sources.list configuration as class file in ${GRML_FAI_CONFIG}/files/etc/apt/sources.list.d/ and adjust the variable -FAI_DEBOOTSTRAP (if not already using a base.tgz) inside -/etc/grml/grml-live.conf[.local]. If you're setting up your own class file don't -forget to include the class name in the class list (grml-live -c ...). +FAI_DEBOOTSTRAP in /etc/grml/grml-live.conf[.local]. If you're setting up +your own class file make sure to include the class name in the class list +(grml-live -c ...). If you want to use a local (for example NFS mount) mirror additionally then adjust MIRROR_DIRECTORY in /etc/grml/grml-live.conf[.local] as well. @@ -695,30 +683,7 @@ How do I create a base tar.gz (I386.tar.gz or AMD64.tar.gz or ARM64.tar.gz) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ [[basetgz]] -First of all create the chroot using debootstrap (requires root): - - BASECHROOT='/tmp/basefile' # path where the chroot gets generated - SUITE='bookworm' # using the current stable release should always work - debootstrap --exclude=info,tasksel,tasksel-data,isc-dhcp-client,isc-dhcp-common "${SUITE}" "${BASECHROOT}" http://deb.debian.org/debian - tar -C "$BASECHROOT" --exclude='var/cache/apt/archives/*.deb' --exclude 'var/lib/apt/lists/*_*' --xattrs --xattrs-include='*.*' --acls -zcf "${SUITE}".tar.gz ./ - -[TIP] -By default debootstrap builds a chroot matching the architecture of the running -host system. If you're using an amd64 system and want to build an i386 base.tgz -then invoke debootstrap using the '--arch i386' option. Disclaimer: building an -AMD64 base.tgz won't work if you are using a 32bit kernel system of course. -Also building an ARM64 base.tgz requires an arm64 system. - -Finally place the generated tarball in $GRML_FAI_CONFIG/basefiles/ (note -that it needs to be uppercase letters matching the class names, so: e.g. -AMD64.tar.gz for amd64, I386.tar.gz for i386 or ARM64.tar.gz for arm64). - -Then executing grml-live should use this file as base system instead of executing -debootstrap. Check out the output for something like: - - [...] - ftar: extracting //srv/config/basefiles///AMD64.tar.gz to /srv/grml64_testing/grml_chroot// - [...] +This is no longer supported. grml-live will call mmdebstrap for you. [[localrepos]] How to use your own local repository @@ -799,7 +764,7 @@ Run grml-live directly from git In case you want to run grml-live directly from the git repository checkout (after making sure all dependencies are installed), you should set -`GRML_FAI_CONFIG` so that a) it finds the according FAI configuration files and +`GRML_FAI_CONFIG` so that a) it finds its class-based configuration files and b) does not use the config files of an possibly installed `grml-live` package. Usage example: diff --git a/etc/grml/grml-live.conf b/etc/grml/grml-live.conf index 748d867d..312622c1 100644 --- a/etc/grml/grml-live.conf +++ b/etc/grml/grml-live.conf @@ -90,12 +90,6 @@ # so you have to create the rootfs structure on your own. # CHROOT_INSTALL="$GRML_FAI_CONFIG/chroot_install" -# Do you want to pass any additional arguments to FAI? -# FAI_ARGS="" - -# Where do you want to store grml-live.log? -# LOGDIR="/var/log/fai/$HOSTNAME/last" - # Which architecture do you want to build? # It defaults to output of 'dpkg --print-architecture' # ARCH="i386" diff --git a/grml-live b/grml-live index fa462c06..ca047231 100755 --- a/grml-live +++ b/grml-live @@ -30,6 +30,14 @@ GRML_LIVE_VERSION='***UNRELEASED***' # global variables PN=$(basename "$0") CMDLINE="$0 $*" +GRML_LIVE_INSTALL_PREFIX=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +if [ -e "${GRML_LIVE_INSTALL_PREFIX}"/usr/lib/grml-live ]; then + # assume source checkout + GRML_LIVE_LIB_DIR="${GRML_LIVE_INSTALL_PREFIX}"/usr/lib/grml-live +else + GRML_LIVE_INSTALL_PREFIX=${GRML_LIVE_INSTALL_PREFIX}/../ + GRML_LIVE_LIB_DIR=${GRML_LIVE_INSTALL_PREFIX}/lib/grml-live +fi # }}} # usage information {{{ @@ -42,9 +50,9 @@ Usage: $PN [options, see as follows] -a architecture; available values: i386, amd64 + arm64 -A clean build directories before and after running - -b build the ISO without updating the chroot via FAI + -b build the ISO without updating the chroot -B build the ISO without touching the chroot (skips cleanup) - -c classes to be used for building the ISO via FAI + -c classes to be used for building the ISO -C configuration file for grml-live -d use specified date instead of build time as date of release -D use specified configuration directory instead of /usr/share/grml-live/config @@ -102,23 +110,8 @@ if [ "$(id -u 2>/dev/null)" != 0 ] ; then exit 1 fi -if [ -r /var/run/fai/FAI_INSTALLATION_IN_PROGRESS ] ; then - echo "/usr/sbin/fai already running or was aborted before.">&2 - echo "You may remove /var/run/fai/FAI_INSTALLATION_IN_PROGRESS and try again.">&2 - exit 1 -fi - -# see #449236 -if [ -r /var/run/fai/fai_softupdate_is_running ] ; then - echo "/usr/sbin/fai softupdate already running or was aborted before.">&2 - echo "You may remove /var/run/fai/fai_softupdate_is_running and try again.">&2 - exit 1 -fi -# }}} - # lsb-functions and configuration stuff {{{ # make sure they are not set by default -VERBOSE='' FORCE='' UPDATE='' BUILD_ONLY='' @@ -127,8 +120,8 @@ BOOTSTRAP_ONLY='' HOSTNAME='' USERNAME='' CONFIGDUMP='' -FAI_CONF_DIR='' -FAI_PROGRAM='fai' +FAI_PROGRAM=$GRML_LIVE_LIB_DIR/minifai +# }}} # don't use colors/escape sequences if [ -r /lib/lsb/init-functions ] ; then @@ -164,24 +157,6 @@ fi # umount all directories {{{ umount_all() { - # make sure we don't leave any mounts - FAI doesn't remove them always - umount "$CHROOT_OUTPUT/proc/sys/fs/binfmt_misc" 2>/dev/null || /bin/true - umount "$CHROOT_OUTPUT/proc" 2>/dev/null || /bin/true - umount "$CHROOT_OUTPUT/run/udev" 2>/dev/null || /bin/true - umount "$CHROOT_OUTPUT/run " 2>/dev/null || /bin/true - umount "$CHROOT_OUTPUT/sys " 2>/dev/null || /bin/true - umount "$CHROOT_OUTPUT/dev/pts" 2>/dev/null || /bin/true - umount "$CHROOT_OUTPUT/dev" 2>/dev/null || /bin/true - - if [ -n "$EXTRACT_ISO_NAME" ] ; then - umount "$EXTRACT_ISO_NAME" 2>/dev/null || /bin/true - fi - - # certain FAI versions sadly leave a ramdisk behind, so better safe than sorry - if [ -x /usr/lib/fai/mkramdisk ] ; then - /usr/lib/fai/mkramdisk -u "$(readlink -f "${CHROOT_OUTPUT}"/var/lib/dpkg)" >/dev/null 2>&1 || /bin/true - fi - [ -n "$MIRROR_DIRECTORY" ] && umount "${CHROOT_OUTPUT}/${MIRROR_DIRECTORY}" } # }}} @@ -193,11 +168,6 @@ store_logfiles() { cp -r "$CHROOT_OUTPUT"/var/log/fai/"$HOSTNAME"/last/* "$LOG_OUTPUT"/fai/ rm -rf "$CHROOT_OUTPUT"/var/log/fai - # store copy of autogenerated configuration files - cp "${FAI_CONF_DIR}"/* "$LOG_OUTPUT"/fai/ - - # copy fai package list - cp "$CHROOT_OUTPUT"/var/log/install_packages.list "$LOG_OUTPUT"/fai/ # fixup owners chown root:adm "$LOG_OUTPUT"/fai/* chmod 664 "$LOG_OUTPUT"/fai/* @@ -206,10 +176,7 @@ store_logfiles() { # clean exit {{{ bailout() { - rm -f /var/run/fai/fai_softupdate_is_running \ - /var/run/fai/FAI_INSTALLATION_IN_PROGRESS [ -n "$CONFIGDUMP" ] && rm -f "$CONFIGDUMP" - [ -n "$FAI_CONF_DIR" ] && rm -rf "$FAI_CONF_DIR" [ -n "$SQUASHFS_STDERR" ] && rm -rf "$SQUASHFS_STDERR" umount_all [ -n "$1" ] && EXIT="$1" || EXIT="1" @@ -265,8 +232,8 @@ extend_string_end() { echo -n "${text}" } -# Returns success if a given fai class was requested. -# This is not called `ifclass`, as fai's ifclass supports a broader syntax. +# Returns success if a given class was requested. +# This is not called `ifclass`, as ifclass supports a broader syntax. hasclass() { local expected_class="$1" case $CLASSES in *,${expected_class},*) return 0 ;; esac @@ -377,7 +344,7 @@ while getopts "a:C:c:d:D:e:g:i:I:o:r:s:S:t:U:v:w:AbBFhnNqQuVz" opt; do F) FORCE=1 ;; u) UPDATE=1 ;; U) CHOWN_USER="$OPTARG" ;; - V) VERBOSE="-v" ;; + V) ;; w) export WAYBACK_DATE="$OPTARG" ;; z) SQUASHFS_ZLIB=1 ;; ?) echo "invalid option -$OPTARG" >&2; usage; bailout 1 ;; @@ -443,7 +410,7 @@ fi [ -n "$NETBOOT" ] || NETBOOT="${OUTPUT}/netboot" # }}} -# some misc checks before executing FAI {{{ +# some misc checks before doing work {{{ [ -n "$CLASSES" ] || bailout 1 "Error: \$CLASSES unset, please set it in $LIVE_CONF or specify it on the command line using the -c option." [ -n "$OUTPUT" ] || bailout 1 "Error: \$OUTPUT unset, please set it in $LIVE_CONF or @@ -466,6 +433,18 @@ if [ -e "$GRML_FAI_CONFIG"/fai.conf ] ; then eend 1 fi +if [ -n "$FAI_ARGS" ] ; then + eerror "The variable \$FAI_ARGS is set, but it is unsupported." ; eend 1 + eerror "Please unset it. Current value: \$FAI_ARGS=$FAI_ARGS" ; eend 1 + bailout +fi + +if [ -n "$FAI_DEBOOTSTRAP_OPTS" ] ; then + eerror "The variable \$FAI_DEBOOTSTRAP_OPTS is set, but it is unsupported." ; eend 1 + eerror "Current value: \$FAI_DEBOOTSTRAP_OPTS=${FAI_DEBOOTSTRAP_OPTS}" ; eend 1 + bailout +fi + if [ -e /etc/grml/fai/config ] && [ -z "$GRML_FAI_CONFIG" ] ; then eerror "Found old configuration files in /etc/grml/fai/config (while \$GRML_FAI_CONFIG was empty)." ; eend 1 eerror "You should check your configuration and move these files into a new path, and set \$GRML_FAI_CONFIG." ; eend 1 @@ -481,10 +460,6 @@ fi # trim characters that are known to cause problems inside $GRML_NAME; # for example isolinux does not like '-' inside the directory name [ -n "$GRML_NAME" ] && SHORT_NAME="$(echo "$GRML_NAME" | tr -d ',./;\- ')" - -# export variables to have them available in fai scripts: -[ -n "$GRML_NAME" ] && export GRML_NAME="$GRML_NAME" -[ -n "$RELEASENAME" ] && export RELEASENAME="$RELEASENAME" # }}} # Warn user if addons from grml-live-addons are absent {{{ @@ -519,11 +494,9 @@ echo " main directory: $OUTPUT" [ -n "$NO_BOOTID" ] && echo " Skipping bootid feature." [ -n "$CHOWN_USER" ] && echo " Output owner: $CHOWN_USER" [ -n "$DEFAULT_BOOTOPTIONS" ] && echo " Adding default bootoptions: \"$DEFAULT_BOOTOPTIONS\"" -[ -n "$FAI_ARGS" ] && echo " Additional arguments for FAI: $FAI_ARGS" [ -n "$LOGFILE" ] && echo " Logging to file: $LOGFILE" [ -n "$SQUASHFS_ZLIB" ] && echo " Using ZLIB (instead of LZMA/XZ) compression." [ -n "$SQUASHFS_OPTIONS" ] && echo " Using SQUASHFS_OPTIONS ${SQUASHFS_OPTIONS}" -[ -n "$VERBOSE" ] && echo " Using VERBOSE mode." [ -n "$CLEAN_ARTIFACTS" ] && echo " Will clean output before and after running." [ -n "$UPDATE" ] && echo " Executing UPDATE instead of fresh installation." if [ -n "$BOOTSTRAP_ONLY" ] ; then @@ -568,26 +541,12 @@ mkdir -p "$(dirname "${LOGFILE}")" touch "$LOGFILE" chown root:adm "$LOGFILE" chmod 664 "$LOGFILE" -# }}} - -# clean/zero/remove logfiles {{{ if [ -n "$PRESERVE_LOGFILE" ] ; then echo "Preserving logfile $LOGFILE as requested via \$PRESERVE_LOGFILE" else # make sure it is empty echo -n > "$LOGFILE" fi - -if [ -n "$ZERO_FAI_LOGFILE" ] ; then - if [ -d /var/log/fai/"$HOSTNAME" ] ; then - rm -rf /var/log/fai/"$HOSTNAME"/"$(readlink /var/log/fai/"$HOSTNAME"/last)" - rm -rf /var/log/fai/"$HOSTNAME"/"$(readlink /var/log/fai/"$HOSTNAME"/last-dirinstall)" - rm -rf /var/log/fai/"$HOSTNAME"/"$(readlink /var/log/fai/"$HOSTNAME"/last-softupdate)" - rm -f /var/log/fai/"$HOSTNAME"/last \ - /var/log/fai/"$HOSTNAME"/last-dirinstall \ - /var/log/fai/"$HOSTNAME"/last-softupdate - fi -fi # }}} # source config and startup {{{ @@ -712,14 +671,7 @@ if [[ -n "${BOOT_METHOD:-}" ]] && [[ "${BOOT_METHOD}" != "isolinux" ]] ; then bailout fi -# generate FAI configuration on the fly -FAI_CONF_DIR=$(mktemp -d) - -echo "# This is an automatically generated file by grml-live. -# Do NOT edit this file, your changes will be lost. -LOGUSER= -# EOF " > "${FAI_CONF_DIR}/fai.conf" - +# continue to support $FAI_DEBOOTSTRAP to specify the Debian suite and mirror URL. if [ -z "$FAI_DEBOOTSTRAP" ] ; then if [ -n "$WAYBACK_DATE" ] ; then FAI_DEBOOTSTRAP="$SUITE http://snapshot.debian.org/archive/debian/$WAYBACK_DATE/" @@ -727,16 +679,6 @@ if [ -z "$FAI_DEBOOTSTRAP" ] ; then FAI_DEBOOTSTRAP="$SUITE http://deb.debian.org/debian" fi fi - -if [ -z "$FAI_DEBOOTSTRAP_OPTS" ] ; then - FAI_DEBOOTSTRAP_OPTS="--exclude=info,tasksel,tasksel-data,isc-dhcp-client,isc-dhcp-common --include=aptitude --arch $ARCH" -fi - -echo "# This is an automatically generated file by grml-live. -# Do NOT edit this file, your changes will be lost. -FAI_DEBOOTSTRAP=\"$FAI_DEBOOTSTRAP\" -FAI_DEBOOTSTRAP_OPTS=\"$FAI_DEBOOTSTRAP_OPTS\" -# EOF " > "${FAI_CONF_DIR}/nfsroot.conf" # }}} # CHROOT_OUTPUT - execute FAI {{{ @@ -762,8 +704,8 @@ else fi if [ -d "$CHROOT_OUTPUT/bin" ] && [ -z "$UPDATE" ] && [ -z "$BUILD_ONLY" ] ; then - log "Skipping stage 'fai dirinstall' as $CHROOT_OUTPUT exists already." - ewarn "Skipping stage 'fai dirinstall' as $CHROOT_OUTPUT exists already." ; eend 0 + log "Skipping stage 'fai $FAI_ACTION' as $CHROOT_OUTPUT exists already." + ewarn "Skipping stage 'fai $FAI_ACTION' as $CHROOT_OUTPUT exists already." ; eend 0 else mkdir -p "$CHROOT_OUTPUT" || bailout 5 "Problem with creating $CHROOT_OUTPUT for FAI" @@ -776,11 +718,9 @@ else mv "${OUTPUT}"/grml_sources "${CHROOT_OUTPUT}"/grml-live/ log "Executed FAI command line:" - log "GRML_LIVE_CONFIG=$CONFIGDUMP $FAI_PROGRAM $VERBOSE -C $FAI_CONF_DIR -s file:///$GRML_FAI_CONFIG -c$CLASSES -u $HOSTNAME $FAI_ACTION $CHROOT_OUTPUT $FAI_ARGS" - # shellcheck disable=SC2086 # $FAI_ARGS needs splitting - GRML_LIVE_CONFIG="$CONFIGDUMP" "$FAI_PROGRAM" $VERBOSE \ - -C "$FAI_CONF_DIR" -s "file:///$GRML_FAI_CONFIG" -c"$CLASSES" \ - -u "$HOSTNAME" "$FAI_ACTION" "$CHROOT_OUTPUT" $FAI_ARGS | tee -a "$LOGFILE" 2>&1 + log "${FAI_PROGRAM} ${GRML_FAI_CONFIG} ${CLASSES} ${HOSTNAME} ${FAI_ACTION} ${CHROOT_OUTPUT} ${CONFIGDUMP} ${FAI_DEBOOTSTRAP}" + # shellcheck disable=SC2086 # $FAI_DEBOOTSTRAP needs splitting + "${FAI_PROGRAM}" "${GRML_FAI_CONFIG}" "${CLASSES}" "${HOSTNAME}" "${FAI_ACTION}" "${CHROOT_OUTPUT}" "${CONFIGDUMP}" ${FAI_DEBOOTSTRAP} | tee -a "${LOGFILE}" 2>&1 RC="${PIPESTATUS[0]}" # notice: bash-only if [ "$RC" != 0 ] ; then @@ -806,49 +746,14 @@ else umount_all - # notice: 'fai dirinstall' does not seem to exit appropriate, so: - ERROR='' - CHECKLOG="$LOG_OUTPUT"/fai/ - if [ -r "$CHECKLOG/software.log" ] ; then - # 1 errors during executing of commands - grep 'dpkg: error processing' "$CHECKLOG/software.log" >> "$LOGFILE" && ERROR=1 - grep 'E: Method http has died unexpectedly!' "$CHECKLOG/software.log" >> "$LOGFILE" && ERROR=2 - grep 'ERROR: chroot' "$CHECKLOG/software.log" >> "$LOGFILE" && ERROR=3 - grep 'E: Failed to fetch' "$CHECKLOG/software.log" >> "$LOGFILE" && ERROR=4 - grep 'Unable to write mmap - msync (28 No space left on device)' "$CHECKLOG/software.log" >> "$LOGFILE" && ERROR=5 - fi - - # FAI versions <6.0 used to write to shell.log - if [ -r "$CHECKLOG/shell.log" ] ; then - grep 'FAILED with exit code' "$CHECKLOG/shell.log" >> "$LOGFILE" && ERROR=6 - fi - - # FAI versions >=6.0 always writes to scripts.log - if [ -r "$CHECKLOG/scripts.log" ] ; then - grep 'FAILED with exit code' "$CHECKLOG/scripts.log" >> "$LOGFILE" && ERROR=6 - fi - - if [ -r "$CHECKLOG/fai.log" ] ; then - grep 'updatebase.*FAILED with exit code' "$CHECKLOG/fai.log" >> "$LOGFILE" && ERROR=7 - grep 'instsoft.*FAILED with exit code' "$CHECKLOG/fai.log" >> "$LOGFILE" && ERROR=8 - fi - - if [ -n "$ERROR" ] ; then - log "Error: there was a critical error [${ERROR}] during execution of stage 'fai dirinstall' [$(date)]" - eerror "Error: there was a critical error during execution of stage 'fai dirinstall'" - eerror "Note: check out ${CHECKLOG}/ for details. [exit ${ERROR}]" - eend 1 - bailout 1 - else - log "Finished execution of stage 'fai dirinstall' [$(date)]" - einfo "Finished execution of stage 'fai dirinstall'" - fi + log "Finished execution of stage 'fai $FAI_ACTION' [$(date)]" + einfo "Finished execution of stage 'fai $FAI_ACTION'" fi fi # BUILD_DIRTY? # }}} # package validator {{{ -CHECKLOG=/var/log/fai/$HOSTNAME/last +CHECKLOG="$LOG_OUTPUT"/fai/ if [ -r "$CHECKLOG/dpkg.selections" ] ; then package_count=$(wc -l "$CHECKLOG/dpkg.selections" | awk '{print $1}') else @@ -1543,8 +1448,8 @@ generate_build_info() { distri_info="${DISTRI_INFO}" \ distri_name="${DISTRI_NAME}" \ extract_iso_name="${EXTRACT_ISO_NAME}" \ - fai_cmdline="BUILD_ONLY=${BUILD_ONLY} BOOTSTRAP_ONLY=${BOOTSTRAP_ONLY} GRML_LIVE_CONFIG=${CONFIGDUMP} WAYBACK_DATE=${WAYBACK_DATE} fai ${VERBOSE} -C ${FAI_CONF_DIR} -s file:///${GRML_FAI_CONFIG} -c${CLASSES} -u ${HOSTNAME} ${FAI_ACTION} ${CHROOT_OUTPUT} ${FAI_ARGS}" \ - fai_version="$(fai --help 2>/dev/null | head -1 | awk '{print $2}' | sed 's/\.$//' || true)" \ + fai_cmdline="${FAI_PROGRAM} ${GRML_FAI_CONFIG} ${CLASSES} ${HOSTNAME} ${FAI_ACTION} ${CHROOT_OUTPUT} ${CONFIGDUMP} ${FAI_DEBOOTSTRAP}" \ + fai_version="minifai" \ grml_architecture="${ARCH}" \ grml_bootid="${BOOTID}" \ grml_build_output="${BUILD_OUTPUT}" \ diff --git a/usr/lib/grml-live/minifai b/usr/lib/grml-live/minifai new file mode 100755 index 00000000..df3a6817 --- /dev/null +++ b/usr/lib/grml-live/minifai @@ -0,0 +1,635 @@ +#!/usr/bin/env python3 +# +# This is a spaghetti-code minimal reimplementation of the FAI API surface grml-live needs, +# for building Grml Live Linux. If you have additional API surface needs, please contribute. +# Please beware that this implementation is an interim step, and we may or may not continue +# with the FAI API. +# +from pathlib import Path +import argparse +import contextlib +import subprocess +import os +import shutil +import socket +import traceback +import sys +import tempfile +from dataclasses import dataclass +from threading import Event, Thread + + +class FaiScriptFailed(Exception): + pass + + +class ClassFileParsingFailed(Exception): + pass + + +@dataclass +class DynamicState: + """Holds state that can change in FAI hooks, for example by calling "skiptask".""" + skip_tasks = set() + + +class PackageList(dict): + def list_for_arch(self, arch: str): + return self.list_of_arch("all") | self.list_of_arch(arch) + + def list_of_arch(self, arch: str): + return set(self.get(arch, [])) + + def as_apt_params(self) -> str: + full_list = [] + for arch, packages in self.items(): + if arch == "all": + full_list += packages + else: + full_list += [f"{package} [{arch}]" for package in packages] + return ", ".join(full_list) + + def merge(self, other) -> None: + for arch, packages in other.items(): + self.setdefault(arch, []) + self[arch] = list(set(self[arch] + packages)) + + +def parse_class_packages(conf_dir: Path, class_name: str) -> PackageList: + """Parse FAI package_config for class class_name.""" + + packagelist = conf_dir / "package_config" / class_name + if not packagelist.exists(): + print(f"I: skipping non-existing {packagelist}", file=sys.stderr) + return PackageList({}) + + print(f"I: parsing {packagelist}") + + arch = "all" + packages = [] + parsed = PackageList({}) + for line in packagelist.read_text().splitlines(): + parts = line.split() + + for index, part in enumerate(parts): + if part.startswith('#'): + parts = parts[0:index] + break + + if not parts: + continue + + if parts[0] == "PACKAGES": + # section header + if len(parts) not in (2, 3): + raise ValueError(f"package class file {packagelist} has invalid PACKAGES line: {line!r}") + if parts[1] != "install": + raise ValueError(f"package class file {packagelist} PACKAGES line not understood: {line!r}") + + # save previously parsed packages + parsed.setdefault(arch, []) + parsed[arch] += packages + + if len(parts) == 3: + arch = parts[2].lower() + else: + arch = "all" + packages = [] + continue + + else: + for part in parts: + if part: + packages.append(part) + + parsed.setdefault(arch, []) + parsed[arch] += packages + return parsed + + +def run_x(args, check: bool = True, **kwargs): + """Run program. Output goes to stdout/stderr.""" + # str-ify Paths, not necessary, but for readability in logs. + args = [arg if isinstance(arg, str) else str(arg) for arg in args] + args_str = '" "'.join(args) + print(f'D: Running "{args_str}"', flush=True) + return subprocess.run(args, check=check, **kwargs) + + +def run_chrooted(chroot_dir: Path, args, check: bool = True, **kwargs): + """Run program with arguments in chroot chroot_dir.""" + kwargs["env"] = { + "PATH": "/usr/sbin:/sbin:/usr/bin:/bin", + "TERM": "dumb", + } | kwargs.get("env", {}) + return run_x(["chroot", chroot_dir] + args, check=check, **kwargs) + + +def chrooted_apt_satisfy(chroot_dir: Path, satisfy_list: str): + """Run apt satisfy in chroot_dir.""" + env = { + "DEBIAN_FRONTEND": "noninteractive", + } + run_chrooted(chroot_dir, [ + "apt", + "satisfy", + "-q", + "-y", + "--no-install-recommends", + satisfy_list + ], env=env, stdin=subprocess.DEVNULL) + + +def chrooted_debconf_set_selections(chroot_dir: Path, selections_file: Path): + """Run debconf-set-selections in chroot_dir, piping in selections_file.""" + + env = { + "DEBIAN_FRONTEND": "noninteractive", + } + print("I: Loading debconf selections from", selections_file) + with selections_file.open("r") as selections_fd: + run_chrooted(chroot_dir, [ + "debconf-set-selections", + "-v" + ], env=env, stdin=selections_fd) + + +def run_fai_script(chroot_dir: Path, script: Path, helper_tools_path: Path, env: dict[str, str]): + """ + Run a FAI hook script or class script, if it exists. + PATH will include helper_tools_path. + Environment will include env. + """ + + if not script.exists(): + print(f"I: Skipping {script}") + return + env = { + "target": str(chroot_dir), + "ROOTCMD": f"chroot {chroot_dir!s} ", + "PATH": str(helper_tools_path) + ":" + os.environ["PATH"], + } | env + print() + print(f"I: *** Running FAI script {script} ... ***") + proc = run_x([script], check=False, env=env, stdin=subprocess.DEVNULL) + if proc.returncode != 0: + print(f"E: FAI script {script} failed with exitcode {proc.returncode} - aborting.") + raise FaiScriptFailed() + print(f"I: Finished FAI script {script}.") + + +def run_class_scripts(conf_dir: Path, chroot_dir: Path, class_name: str, helper_tools_path: Path, env: dict[str, str]): + print() + print(f"I: Running \"scripts\" for class {class_name}...") + print() + scripts_dir = conf_dir / "scripts" / class_name + for script in sorted(scripts_dir.glob("*")): + if script.name.endswith(".dpkg-old") or script.name.endswith(".dpkg-new"): + print(f"W: skipping {script} due to name suffix, please delete it") + continue + run_fai_script(chroot_dir, script, helper_tools_path, env) + + +def install_packages_for_classes(conf_dir: Path, chroot_dir: Path, classes: list[str], dynamic_state: DynamicState): + """Run equivalent of "instsoft" task: set debconf selections and install packages listed in package lists.""" + + # debconf is not Essential. Ensure it is installed, so we can use debconf-set-selections. + chrooted_apt_satisfy(chroot_dir, "debconf") + for class_name in classes: + selections_file = conf_dir / "debconf" / class_name + if not selections_file.exists(): + print(f"I: skipping non-existing {selections_file}", file=sys.stderr) + continue + chrooted_debconf_set_selections(chroot_dir, selections_file) + + full_package_list = PackageList() + for class_name in classes: + package_list = parse_class_packages(conf_dir, class_name) + full_package_list.merge(package_list) + satisfy_args = package_list.as_apt_params() + if satisfy_args: + print() + print(f"I: Installing packages for class {class_name}") + chrooted_apt_satisfy(chroot_dir, satisfy_args) + + print() + print("I: Installing packages based on merged package list") + chrooted_apt_satisfy(chroot_dir, full_package_list.as_apt_params()) + + +def show_env(log_text: str, env): + print(f"D: Showing {log_text} ...") + for k, v in dict(env).items(): + print(f"D: {log_text}: {k}={v}") + print() + + +def do_fcopy_path(source_dir: Path, dest_path: Path, classes: list[str], mode) -> bool: + print(f"D: fcopy_path {source_dir} {dest_path} {mode=}") + + to_copy = None + for class_name in classes: + if (source_dir / class_name).exists(): + to_copy = source_dir / class_name + + if to_copy is not None: + print(f"I: fcopy: writing {to_copy} to {dest_path}...") + if dest_path.exists(): + print(f"W: destination {dest_path} already exists - behaviour undefined") + + # this is probably fine, as we expect to run as root and do not support + # different file/directory ownership. + dest_path.parent.mkdir(exist_ok=True, parents=True) + + shutil.copyfile(to_copy, dest_path, follow_symlinks=False) + dest_path.chmod(mode) + os.chown(dest_path, 0, 0, follow_symlinks=False) + + return True + + +def parse_fcopy_args(fcopy_args: list[str]) -> tuple[str, str, int, bool, bool, list[str]]: + user = "root" + group = "root" + mode = 0o644 + recursive = False + ignore_missing = False + paths = [] + + """ + -B Remove backup files with suffix .pre_fcopy. You can also set the environment variable FCOPY_NOBACKUP to 1. + -r Copy recursively (traverse down the tree). Copy all files below SOURCE. These are all subdirectory leaves in the SOURCE tree. Ignore "ignored" directories (see "-I" for details). + -i Ignore warnings about no matching class and non-existing source directories. These warnings will not set the exit code to 1. + -v verbose + -m user,group,mode Set user, group and mode for all copied files (mode as octal number, user and group numeric id or name). If not specified, use file file-modes or data of source file. + -M Use default values for user, group and mode. This is equal to -m root,root,0644 + """ + parse_m = False + for index, arg in enumerate(fcopy_args): + if parse_m: + # TODO: handle errors + user, group, mode = arg.split(",") + mode = int(mode, 8) + parse_m = False + elif arg in ["-M", "-B", "-v"]: + # defaulted / ignored + pass + elif arg == "-m": + parse_m = True + elif arg == "-r": + recursive = True + elif arg == "-i": + ignore_missing = True + elif not arg.startswith("-"): + paths = fcopy_args[index:] + break + else: + raise ValueError(f"do_fcopy: param {arg} not understood") + + if not paths: + raise ValueError("do_fcopy: no paths given") + + paths = [path.lstrip("/") for path in paths] + + return user, group, mode, recursive, ignore_missing, paths + + +def do_fcopy(conf_dir: Path, chroot_dir: Path, classes: list[str], fcopy_args: list[str]) -> int: + try: + user, group, mode, recursive, ignore_missing, paths = parse_fcopy_args(fcopy_args) + if user != "root" or group != "root": + raise ValueError("E: cannot set user/group to something else than root") + except Exception as except_inst: + print(f"E: parsing fcopy_args {fcopy_args!r} failed: {except_inst}") + return 1 + + print(f"D: fcopy {recursive=} {ignore_missing=} {user=} {group=} {mode=} {paths=}") + rc = 0 + files_dir = conf_dir / "files" + + if recursive: + for path in paths: + path_dir = files_dir / path + dirs = [p for p in path_dir.glob("**/*") if p.is_dir()] + for dir in dirs: + dir = dir.relative_to(files_dir) + do_fcopy_path(files_dir / dir, chroot_dir / dir, classes, mode) + + else: + for path in paths: + path_dir = files_dir / path + found = False + if path_dir.exists(): + found = do_fcopy_path(path_dir, chroot_dir / path, classes, mode) + if not found and not ignore_missing: + print(f"E: do_fcopy: source {path=} is missing") + rc = 1 + + return rc + + +def do_skiptask(dynamic_state: DynamicState, skiptask_args: list[str]): + print(f"I: requesting skipping of tasks: {' '.join(skiptask_args)}") + dynamic_state.skip_tasks.update(skiptask_args) + + +def helper_socket_thread(tempdir: Path, conf_dir: Path, chroot_dir: Path, classes: list[str], exit_event: Event, dynamic_state: DynamicState): + address_family = socket.AF_UNIX + socket_type = socket.SOCK_STREAM + request_queue_size = 5 + + listen_socket = socket.socket(address_family, socket_type) + listen_socket.bind(f"{tempdir}/sock") + listen_socket.listen(request_queue_size) + listen_socket.settimeout(1) + + while not exit_event.is_set(): + try: + request_socket, _ = listen_socket.accept() + except TimeoutError: + continue + + orig_req = request_socket.recv(4096).decode() + req = orig_req.split("\n") + rc = 120 + if len(req) != 2 and req[1] != "": + print(">>socket thread>>got>>", repr(orig_req)) + print(">>socket thread>>no endl, req truncated?") + else: + req = req[0].split(" ") + if req[0] == "fcopy": + rc = do_fcopy(conf_dir, chroot_dir, classes, req[1:]) + elif req[0] == "skiptask": + rc = do_skiptask(dynamic_state, req[1:]) + else: + print(">>socket thread>>req not understood:", repr(orig_req)) + + request_socket.send(f"{rc!s}\n".encode()) + request_socket.close() + + listen_socket.close() + + +def write_helper_tool(tools_path: Path, tool_name: str, body: str): + with (tools_path / tool_name).open("wt") as file: + file.write(body) + os.fchmod(file.fileno(), 0o755) + + +@contextlib.contextmanager +def helper_tools(conf_dir: Path, chroot_dir: Path, classes: list[str], dynamic_state: DynamicState): + tempdir = Path(tempfile.mkdtemp()) + + write_helper_tool(tempdir, "fcopy", f"""#!/bin/sh +# TODO: -t 300 is a hack that needs to be fixed somehow +RC=$(echo fcopy "$@" | socat -t 300 - UNIX-CONNECT:{tempdir}/sock) +if [ -z "$RC" ]; then + echo "E: minifai fcopy: got no reply from server" + exit 119 +elif [ "$RC" != "0" ]; then + echo "E: minifai fcopy: server sent error code $RC" + exit "$RC" +fi +exit 0 +""") + + write_helper_tool(tempdir, "ifclass", f"""#!/bin/bash +haystack=:{":".join(classes)}: +if [[ ":$haystack:" = *:$1:* ]]; then + echo "I: ifclass: yes, having class $1" + exit 0 +else + echo "I: ifclass: no, not having class $1" + exit 1 +fi +""") + + write_helper_tool(tempdir, "skiptask", f"""#!/bin/sh +echo skiptask "$@" | socat - UNIX-CONNECT:{tempdir}/sock,forever +exit 0 + """) + + exit_event = Event() + thread = Thread(target=helper_socket_thread, args=(tempdir, conf_dir, chroot_dir, classes, exit_event, dynamic_state), daemon=False) + thread.start() + try: + yield tempdir + finally: + exit_event.set() + thread.join() + shutil.rmtree(tempdir, ignore_errors=True) + + +@contextlib.contextmanager +def policy_rcd(chroot_dir: Path): + marker = "!MINIFAI!" + print("I: installing temporary policy-rc.d") + program = chroot_dir / "usr" / "sbin" / "policy-rc.d" + with program.open("wt") as file: + file.write(f"#!/bin/sh\n# Installed by grml-live minifai {marker}\nexit 101\n") + os.fchmod(file.fileno(), 0o755) + + try: + yield + finally: + try: + if marker in program.read_text(): + program.unlink() + else: + print(f"I: not cleaning up {program} - our marker went missing") + except: + print(f"W: failed cleaning up {program}") + + +def create_logdir(chroot_dir: Path, hostname: str) -> Path: + fai_log_dir = chroot_dir / "var" / "log" / "fai" + fai_log_dir.mkdir(exist_ok=True) + hostname_log_dir = fai_log_dir / hostname + hostname_log_dir.mkdir(exist_ok=True) + last_log_dir = hostname_log_dir / "last" + if last_log_dir.exists(): + print(f"I: deleting log directory from previous run: {last_log_dir}") + shutil.rmtree(last_log_dir) + print(f"I: creating {last_log_dir}") + last_log_dir.mkdir() + + # Create a file in there, so grml-live does not complain. + (last_log_dir / "minifai").write_text( + "This chroot was created by grml-live minifai. Not all features are supported.\n" + ) + + return last_log_dir + + +def parse_varfile(varfile: Path) -> dict: + env = {} + lines = varfile.read_text().splitlines() + for lineno, orig_line in enumerate(lines): + # strip off comments + line = orig_line.split('#', 1) + if len(line) == 2: + line = line[0].rstrip() + elif line[0] == '': + line = '' + else: + line = line[0] + + if not line: + continue + + line = line.split('=', 1) + if len(line) == 2: + k, v = line + v = v.lstrip() + if (v.startswith('"') and v.endswith('"')) or (v.startswith("'") and v.endswith("'")): + v = v[1:-1] + + k = k.rstrip() + if not k.startswith(' '): + # TODO: should instead check if k starts with alphanumeric (or whatever is allowed) + env[k] = v + continue + + print(f"E: could not understand line {lineno} in {varfile}: {orig_line}") + raise ClassFileParsingFailed() + + return env + + +def read_vars_for_classes(conf_dir: Path, classes: list[str]) -> dict: + """Parse FAI .var files""" + env = {} + + for class_name in classes: + varfile = conf_dir / "class" / f"{class_name}.var" + if varfile.exists(): + env.update(parse_varfile(varfile)) + + return env + + +def install_base(chroot_dir: Path, debian_suite: str, mirror_url: str): + print("I: Installing Debian base using mmdebstrap") + included_packages = ["netbase"] + if mirror_url.startswith("https"): + included_packages.append("ca-certificates") + run_x(["ls", "-laR", chroot_dir]) + args = [ + "mmdebstrap", + "--format=directory", + "--variant=required", + "--verbose", + "--skip=check/empty", # grml-live pre-creates directories in chroot + f"--include={','.join(included_packages)}", + debian_suite, + chroot_dir, + mirror_url, + ] + run_x(args) + + +def should_skip_task(dynamic_state: DynamicState, task: str) -> bool: + if task in dynamic_state.skip_tasks: + print(f"I: skipping FAI task \"{task}\", as requested") + return True + return False + + +def task_updatebase(chroot_dir: Path, dynamic_state: DynamicState): + if should_skip_task(dynamic_state, "updatebase"): + return + run_chrooted(chroot_dir, ["apt-get", "update", "-q"]) + + +def _run_tasks(conf_dir: Path, chroot_dir: Path, classes: list[str], hostname: str, grml_live_config: Path, fai_action: str) -> int: + dynamic_state = DynamicState() + logdir = create_logdir(chroot_dir, hostname) + + env = { + "GRML_LIVE_CONFIG": str(grml_live_config), + "LOGDIR": str(logdir), + } | read_vars_for_classes(conf_dir, classes) + show_env("merged class variables", env) + + with helper_tools(conf_dir, chroot_dir, classes, dynamic_state) as helper_tools_path: + hook_env = env | {"FAI_ACTION": fai_action} + for class_name in classes: + run_fai_script(chroot_dir, conf_dir / "hooks" / f"updatebase.{class_name}", helper_tools_path, hook_env) + + with policy_rcd(chroot_dir): + task_updatebase(chroot_dir, dynamic_state) + + if not should_skip_task(dynamic_state, "instsoft"): + for class_name in classes: + run_fai_script(chroot_dir, conf_dir / "hooks" / f"instsoft.{class_name}", helper_tools_path, hook_env) + + install_packages_for_classes(conf_dir, chroot_dir, classes, dynamic_state) + + if not should_skip_task(dynamic_state, "configure"): + for class_name in classes: + run_class_scripts(conf_dir, chroot_dir, class_name, helper_tools_path, env) + + run_chrooted(chroot_dir, ["ls", "-laR", "/dev/"]) + + return 0 + + +def create_argparser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + # path to fai classes, scripts, ... + parser.add_argument("config", type=Path) + parser.add_argument("classes") + parser.add_argument("hostname") + parser.add_argument("action", choices=["dirinstall", "softupdate"]) + parser.add_argument("chroot_dir", type=Path) + parser.add_argument("grml_live_config", type=Path) + parser.add_argument("debian_suite", type=str) + parser.add_argument("mirror_url", type=str) + return parser + + +def main(program_name: str, argv: list[str]) -> int: + print(f"I: {program_name} started with {argv=}") + show_env("minifai environment", os.environ) + args = create_argparser().parse_args() + print(f"I: config: {args.config}") + classes = args.classes.split(",") + print(f"I: classes: {classes}") + print(f"I: hostname: {args.hostname}") + print(f"I: action: {args.action}") + conf_dir = args.config.absolute() + print(f"I: conf_dir: {conf_dir}") + chroot_dir: Path = args.chroot_dir.absolute() + print(f"I: chroot_dir: {args.chroot_dir}") + print(f"I: grml_live_config: {args.grml_live_config}") + print(f"I: debian_suite: {args.debian_suite}") + print(f"I: mirror_url: {args.mirror_url}") + + if not conf_dir.exists(): + raise ValueError(f"config directory {conf_dir} does not exist") + if not chroot_dir.exists(): + raise ValueError(f"chroot directory {chroot_dir} does not exist") + + try: + if args.action == "dirinstall": + install_base(conf_dir, chroot_dir, classes) + rc = _run_tasks(conf_dir, chroot_dir, classes, args.hostname, args.grml_live_config, args.action) + elif args.action == "softupdate": + rc = _run_tasks(conf_dir, chroot_dir, classes, args.hostname, args.grml_live_config, args.action) + else: + print(f"E: minifai: Unknown fai action: {args.action!r}") + rc = 1 + except (ClassFileParsingFailed, FaiScriptFailed): + # assume exception site already printed relevant info + rc = 3 + except Exception: + print("E: Caught fatal exception") + traceback.print_exc() + rc = 2 + + print(f"I: minifai exiting with exit code {rc}") + return rc + + +if __name__ == "__main__": + sys.exit(main(sys.argv.pop(0), sys.argv))