POC: Dynamically load ignition and afterburn from rootfs

This is a proof-of-concept of the idea in
coreos/fedora-coreos-tracker#1247 (comment)

The role of the initramfs originally was just to mount the root filesystem.
Us running ignition from the initramfs makes sense, but it doesn't mean
the ignition binary has to physically live in the initramfs.

In the end state our initramfs for example doesn't need to physically
contain NetworkManager for example either. Or for that matter, kernel network drivers.

It just has to have enough code to mount the root filesystem, and
neither ignition nor afterburn are needed for that.

This clearly adds some nontrivial logic to our already nontrivial
initramfs.  But, it does shave 9M from each copy of the initramfs,
so in the likely case of having (transiently) 3 different versions,
we will save 27MB in /boot, which is a good amount.

overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-firstboot-populate-initramfs.service

@@ -0,0 +1,28 @@
+[Unit]
+Description=Ignition OSTree: Populate initramfs for Ignition
+DefaultDependencies=false
+# Any services looking at mounts need to order after (or before) this
+# because it causes device re-probing.
+Before=coreos-gpt-setup.service
+# Also this one is probing disks
+Before=coreos-unique-boot.service
+# On the ignition boot, we must have a device labeled `root`.
+Wants=systemd-udevd.service
+After=systemd-udevd.service
+Requires=dev-disk-by\x2dlabel-root.device
+After=dev-disk-by\x2dlabel-root.device
+# We need to run before any services which execute:
+# - /usr/bin/ignition
+# - /usr/bin/afterburn
+Before=ignition-fetch.service
+Before=afterburn-network-kargs.service
+ConditionKernelCommandLine=ostree
+OnFailure=emergency.target
+OnFailureJobMode=isolate
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+# So we can transiently mount sysroot
+MountFlags=slave
+ExecStart=/usr/libexec/ignition-ostree-transposefs populate-initramfs-pre-ignition

overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-subsequent-populate-initramfs.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=Ignition OSTree: Populate initramfs (subsequent boot)
+After=initrd-root-fs.target
+# We need to run before any services which execute:
+# - /usr/bin/ignition
+# - /usr/bin/afterburn
+Before=afterburn-hostname.service
+# This ensures we don't duplicate work on firstboot
+ConditionPathExists=!/usr/bin/ignition
+ConditionKernelCommandLine=ostree
+OnFailure=emergency.target
+OnFailureJobMode=isolate
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+# So we can transiently mount sysroot
+MountFlags=slave
+ExecStart=/usr/libexec/ignition-ostree-transposefs populate-initramfs-subsequent

overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-transposefs.sh

@@ -5,6 +5,10 @@ boot_sector_size=440
 esp_typeguid=c12a7328-f81f-11d2-ba4b-00a0c93ec93b
 bios_typeguid=21686148-6449-6e6f-744e-656564454649
 prep_typeguid=9e1a2d38-c612-4316-aa26-8b49521e5a8b
+# Binaries we store in the real root; keep this list in sync
+# with what's in module-setup.sh to remove them.
+rootfs_initramfs_binaries=(/usr/bin/afterburn)
+rootfs_ignition=/usr/lib/dracut/modules.d/30ignition/ignition
 
 # This is implementation details of Ignition; in the future, we should figure
 # out a way to ask Ignition directly whether there's a filesystem with label
@@ -84,6 +88,25 @@ mount_and_restore_filesystem_by_label() {
 }
 
 case "${1:-}" in
+    # See documentation in the unit for this
+    populate-initramfs-pre-ignition)
+        # We're running before Ignition; we must mount sysroot on our own and find
+        # the ostree deployment, similar to the transposefs-save case.
+        mount_verbose "${root_part}" /sysroot
+        deployment=$(ls -d /sysroot/ostree/deploy/*/deploy/*/)
+        for binary in ${rootfs_initramfs_binaries[@]}; do
+            cp -p ${deployment}${binary} ${binary}
+        done
+        cp -p ${deployment}${rootfs_ignition} /usr/bin/ignition
+        ;;
+    # See documentation in the unit for this
+    populate-initramfs-subsequent)
+        for binary in ${rootfs_initramfs_binaries[@]}; do
+            # Here we have /sysroot mounted already, so just copy from there.
+            cp -p /sysroot${binary} ${binary}
+        done
+        cp -p /sysroot${rootfs_ignition} /usr/bin/ignition
+        ;;
     detect)
         # Mounts are not in a private namespace so we can mount ${saved_data}
         wipes_root=$(jq "$(query_fslabel root) | length" "${ignition_cfg}")

overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/module-setup.sh

@@ -6,13 +6,19 @@ depends() {
     echo ignition rdcore
 }
 
-install_ignition_unit() {
+install_unit() {
     local unit=$1; shift
-    local target=${1:-complete}
+    local target=${1:-initrd}
     inst_simple "$moddir/$unit" "$systemdsystemunitdir/$unit"
     # note we `|| exit 1` here so we error out if e.g. the units are missing
     # see https://github.com/coreos/fedora-coreos-config/issues/799
-    systemctl -q --root="$initdir" add-requires "ignition-${target}.target" "$unit" || exit 1
+    systemctl -q --root="$initdir" add-requires "${target}.target" "$unit" || exit 1
+}
+
+install_ignition_unit() {
+    local unit=$1; shift
+    local target=${1:-complete}
+    install_unit "${unit}" "ignition-${target}"
 }
 
 installkernel() {
@@ -92,6 +98,13 @@ install() {
     inst_script "$moddir/coreos-rootflags.sh" \
         "/usr/sbin/coreos-rootflags"
 
+    # Support for initramfs binaries loaded from the real root
+    # https://github.com/coreos/fedora-coreos-tracker/issues/1247
+    install_ignition_unit ignition-ostree-firstboot-populate-initramfs.service diskful
+    install_unit ignition-ostree-subsequent-populate-initramfs.service initrd
+    # Keep this in sync with ignition-ostree-transposefs.sh
+    rm -v "${initdir}"/usr/bin/{ignition,afterburn}
+
     install_ignition_unit ignition-ostree-growfs.service
     inst_script "$moddir/ignition-ostree-growfs.sh" \
         /usr/sbin/ignition-ostree-growfs