buildextend-installer: move PXE squashfs and osmet to separate initrd

fixtures/fcos.json

@@ -91,6 +91,10 @@
             "path": "fedora-coreos-31.20200310.20.0-live-initramfs.x86_64.img",
             "sha256": "cbbf3a6cf94129afa598fba6b1d0e00738d80b5b43ee462d633ed4b8d59cc11b"
         },
+        "live-rootfs": {
+            "path": "fedora-coreos-31.20200310.20.0-live-rootfs.x86_64.img",
+            "sha256": "dbbf3a6cf94129afa598fba6b1d0e00738d80b5b43ee462d633ed4b8d59cc11b"
+        },
         "azure": {
             "path": "fedora-coreos-31.20200310.20.0-azure.x86_64.vhd.xz",
             "sha256": "4062628000a1e114d3eb7a75332d01ab516445eedacfa5cf5bbdd40c89cb68e1",

mantle/cmd/kola/testiso.go

@@ -60,7 +60,8 @@ var (
 
 	scenarios []string
 
-	pxeKernelArgs []string
+	pxeAppendRootfs bool
+	pxeKernelArgs   []string
 
 	debug bool
 )
@@ -124,6 +125,8 @@ func init() {
 	cmdTestIso.Flags().BoolVarP(&nopxe, "no-pxe", "P", false, "Skip testing live installer PXE")
 	cmdTestIso.Flags().BoolVarP(&noiso, "no-iso", "", false, "Skip testing live installer ISO")
 	cmdTestIso.Flags().BoolVar(&debug, "debug", false, "Display qemu console to stdout, turn off automatic initramfs failure checking")
+	// TODO: default to false once https://github.com/coreos/fedora-coreos-config/pull/528 lands
+	cmdTestIso.Flags().BoolVar(&pxeAppendRootfs, "pxe-append-rootfs", true, "Append rootfs to PXE initrd instead of fetching at runtime")
 	cmdTestIso.Flags().StringSliceVar(&pxeKernelArgs, "pxe-kargs", nil, "Additional kernel arguments for PXE")
 	// FIXME move scenarioISOLiveLogin into the defaults once https://github.com/coreos/fedora-coreos-config/pull/339#issuecomment-613000050 is fixed
 	cmdTestIso.Flags().StringSliceVar(&scenarios, "scenarios", []string{scenarioPXEInstall, scenarioISOOfflineInstall}, fmt.Sprintf("Test scenarios (also available: %v)", []string{scenarioLegacyInstall, scenarioISOLiveLogin, scenarioISOInstall}))
@@ -248,8 +251,9 @@ func runTestIso(cmd *cobra.Command, args []string) error {
 	}
 
 	baseInst := platform.Install{
-		CosaBuild: kola.CosaBuild,
-		Native4k:  kola.QEMUOptions.Native4k,
+		CosaBuild:       kola.CosaBuild,
+		Native4k:        kola.QEMUOptions.Native4k,
+		PxeAppendRootfs: pxeAppendRootfs,
 
 		IgnitionSpec2: kola.Options.IgnitionVersion == "v2",
 	}

mantle/cosa/cosa_v1.go

@@ -77,6 +77,7 @@ type BuildArtifacts struct {
 	LiveInitramfs *Artifact `json:"live-initramfs,omitempty"`
 	LiveIso       *Artifact `json:"live-iso,omitempty"`
 	LiveKernel    *Artifact `json:"live-kernel,omitempty"`
+	LiveRootfs    *Artifact `json:"live-rootfs,omitempty"`
 	Metal         *Artifact `json:"metal,omitempty"`
 	Metal4KNative *Artifact `json:"metal4k,omitempty"`
 	OpenStack     *Artifact `json:"openstack,omitempty"`

mantle/cosa/schema_doc.go

@@ -342,6 +342,7 @@ var generatedSchemaJSON = `{
        "live-kernel",
        "live-initramfs",
        "live-iso",
+       "live-rootfs",
        "metal",
        "metal4k",
        "openstack",
@@ -422,6 +423,12 @@ var generatedSchemaJSON = `{
         "title":"Live ISO",
         "$ref": "#/definitions/artifact"
        },
+       "live-rootfs": {
+         "$id":"#/properties/images/properties/live-rootfs",
+         "type":"object",
+         "title":"Live Rootfs",
+         "$ref": "#/definitions/artifact"
+        },
        "openstack": {
          "$id":"#/properties/images/properties/openstack",
          "type":"object",

mantle/fcos/metadata.go

@@ -143,6 +143,7 @@ type ImageFormat struct {
 	Disk      *ImageType `json:"disk,omitempty"`
 	Kernel    *ImageType `json:"kernel,omitempty"`
 	Initramfs *ImageType `json:"initramfs,omitempty"`
+	Rootfs    *ImageType `json:"rootfs,omitempty"`
 }
 
 // ImageType contains image detail

mantle/platform/metal.go

@@ -16,6 +16,7 @@ package platform
 
 import (
 	"fmt"
+	"io"
 	"io/ioutil"
 	"math"
 	"net"
@@ -96,6 +97,7 @@ type Install struct {
 	IgnitionSpec2   bool
 	LegacyInstaller bool
 	Native4k        bool
+	PxeAppendRootfs bool
 
 	// These are set by the install path
 	kargs        []string
@@ -137,6 +139,7 @@ func (inst *Install) PXE(kargs []string, liveIgnition, ignition ignv3types.Confi
 		mach, err = inst.runPXE(&kernelSetup{
 			kernel:    inst.CosaBuild.Meta.BuildArtifacts.LiveKernel.Path,
 			initramfs: inst.CosaBuild.Meta.BuildArtifacts.LiveInitramfs.Path,
+			rootfs:    inst.CosaBuild.Meta.BuildArtifacts.LiveRootfs.Path,
 		}, false)
 		if err != nil {
 			return nil, errors.Wrapf(err, "testing live installer")
@@ -158,7 +161,7 @@ func (inst *InstalledMachine) Destroy() error {
 }
 
 type kernelSetup struct {
-	kernel, initramfs string
+	kernel, initramfs, rootfs string
 }
 
 type pxeSetup struct {
@@ -275,11 +278,25 @@ func (inst *Install) setup(kern *kernelSetup) (*installerRun, error) {
 		return nil, err
 	}
 
-	for _, name := range []string{kern.kernel, kern.initramfs} {
+	for _, name := range []string{kern.kernel, kern.initramfs, kern.rootfs} {
+		if name == "" {
+			continue
+		}
 		if err := absSymlink(filepath.Join(builddir, name), filepath.Join(tftpdir, name)); err != nil {
 			return nil, err
 		}
 	}
+	if kern.rootfs != "" && inst.PxeAppendRootfs {
+		// replace the initramfs symlink with a concatenation of
+		// the initramfs and rootfs
+		initrd := filepath.Join(tftpdir, kern.initramfs)
+		if err := os.Remove(initrd); err != nil {
+			return nil, err
+		}
+		if err := cat(initrd, filepath.Join(builddir, kern.initramfs), filepath.Join(builddir, kern.rootfs)); err != nil {
+			return nil, err
+		}
+	}
 
 	var metalimg string
 	if inst.Native4k {
@@ -369,6 +386,9 @@ func (t *installerRun) destroy() error {
 }
 
 func (t *installerRun) completePxeSetup(kargs []string) error {
+	if t.kern.rootfs != "" && !t.inst.PxeAppendRootfs {
+		kargs = append(kargs, fmt.Sprintf("coreos.live.rootfs_url=%s/%s", t.baseurl, t.kern.rootfs))
+	}
 	kargsStr := strings.Join(kargs, " ")
 
 	var bootfile string
@@ -437,6 +457,26 @@ func (t *installerRun) completePxeSetup(kargs []string) error {
 	return nil
 }
 
+func cat(outfile string, infiles ...string) error {
+	out, err := os.OpenFile(outfile, os.O_WRONLY|os.O_CREATE, 0644)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+	for _, infile := range infiles {
+		in, err := os.Open(infile)
+		if err != nil {
+			return err
+		}
+		defer in.Close()
+		_, err = io.Copy(out, in)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (t *installerRun) run() (*QemuInstance, error) {
 	builder := t.builder
 	netdev := fmt.Sprintf("%s,netdev=mynet0,mac=52:54:00:12:34:56", t.pxe.networkdevice)

src/cmd-buildextend-installer

@@ -4,6 +4,7 @@
 # An operation that creates an ISO image for installing CoreOS
 
 import argparse
+import hashlib
 import json
 import os
 import re
@@ -45,6 +46,8 @@ parser.add_argument("--fast", action='store_true', default=False,
                     help="Reduce compression for development (FCOS only)")
 parser.add_argument("--force", action='store_true', default=False,
                     help="Overwrite previously generated installer")
+parser.add_argument("--legacy-pxe", action='store_true', default=False,
+                    help="Generate stub PXE rootfs image")
 parser.add_argument("--no-pxe", action='store_true', default=False,
                     help="Do not generate PXE media")
 args = parser.parse_args()
@@ -108,11 +111,15 @@ tmpisoimages = os.path.join(tmpisoroot, 'images')
 tmpisoisolinux = os.path.join(tmpisoroot, 'isolinux')
 # contents of initramfs on both PXE and ISO
 tmpinitrd_base = os.path.join(tmpdir, 'initrd-base')
+# additional contents of initramfs on ISO
+tmpinitrd_iso = os.path.join(tmpdir, 'initrd-iso')
 # additional contents of initramfs on PXE
 tmpinitrd_pxe = os.path.join(tmpdir, 'initrd-pxe')
+# contents of PXE rootfs image
+tmpinitrd_pxe_rootfs = os.path.join(tmpdir, 'initrd-pxe-rootfs')
 
 for d in (tmpdir, tmpisoroot, tmpisoimages, tmpisoisolinux, tmpinitrd_base,
-        tmpinitrd_pxe):
+        tmpinitrd_iso, tmpinitrd_pxe, tmpinitrd_pxe_rootfs):
     os.mkdir(d)
 
 # Number of padding bytes at the end of the ISO initramfs for embedding
@@ -145,6 +152,20 @@ def cp_reflink(src, dest):
     subprocess.check_call(['cp', '--reflink=auto', src, dest])
 
 
+# Make stream hash for `rdcore stream-hash`
+# https://github.com/coreos/coreos-installer/blob/a8d6f50dea6e/src/bin/rdcore/stream_hash.rs#L26-L41
+def make_stream_hash(src, dest):
+    bufsize = 2 * 1024 * 1024
+    with open(src, 'rb') as inf:
+        with open(dest, 'w') as outf:
+            outf.write('stream-hash sha256 {}\n'.format(bufsize))
+            while True:
+                buf = inf.read(bufsize)
+                if not buf:
+                    break
+                outf.write(hashlib.sha256(buf).hexdigest() + '\n')
+
+
 def generate_iso():
     # convention for kernel and initramfs names
     kernel_img = 'vmlinuz'
@@ -182,19 +203,33 @@ def generate_iso():
         os.chmod(os.path.join(tmpisoimages, file), 0o644)
 
     # Generate initramfs stamp file which says whether it's a live or legacy
-    # initramfs.
+    # initramfs.  Store the build ID in it.
     if is_live:
         stampname = 'etc/coreos-live-initramfs'
     else:
         stampname = 'etc/coreos-legacy-installer-initramfs'
     stamppath = os.path.join(tmpinitrd_base, stampname)
     os.makedirs(os.path.dirname(stamppath), exist_ok=True)
-    open(stamppath, 'w').close()
+    with open(stamppath, 'w') as fh:
+        fh.write(args.build + '\n')
+
+    # Generate rootfs stamp file with the build ID, indicating that the
+    # rootfs has been appended and confirming that initramfs and rootfs are
+    # from the same build.
+    if is_live:
+        stamppath = os.path.join(tmpinitrd_pxe_rootfs, 'etc/coreos-live-rootfs')
+        os.makedirs(os.path.dirname(stamppath), exist_ok=True)
+        with open(stamppath, 'w') as fh:
+            fh.write(args.build + '\n')
 
     # Add osmet files
+    if args.legacy_pxe:
+        tmpinitrd_pxe_or_rootfs = tmpinitrd_pxe
+    else:
+        tmpinitrd_pxe_or_rootfs = tmpinitrd_pxe_rootfs
     if is_live:
-        tmp_osmet = os.path.join(tmpinitrd_base, img_metal_obj['path'] + '.osmet')
-        tmp_osmet4k = os.path.join(tmpinitrd_base, img_metal4k_obj['path'] + '.osmet')
+        tmp_osmet = os.path.join(tmpinitrd_iso, img_metal_obj['path'] + '.osmet')
+        tmp_osmet4k = os.path.join(tmpinitrd_iso, img_metal4k_obj['path'] + '.osmet')
         print(f'Generating osmet file for 512b metal image')
         run_verbose(['/usr/lib/coreos-assembler/osmet-pack',
                      img_metal, '512', tmp_osmet, img_metal_checksum,
@@ -203,6 +238,9 @@ def generate_iso():
         run_verbose(['/usr/lib/coreos-assembler/osmet-pack',
                      img_metal4k, '4096', tmp_osmet4k, img_metal4k_checksum,
                      'fast' if args.fast else 'normal'])
+        if not args.no_pxe:
+            cp_reflink(tmp_osmet, os.path.join(tmpinitrd_pxe_or_rootfs, os.path.basename(tmp_osmet)))
+            cp_reflink(tmp_osmet4k, os.path.join(tmpinitrd_pxe_or_rootfs, os.path.basename(tmp_osmet4k)))
 
     # Generate root squashfs
     tmp_squashfs = None
@@ -212,7 +250,7 @@ def generate_iso():
         run_verbose(['/usr/lib/coreos-assembler/gf-mksquashfs',
                     img_metal, tmp_squashfs, squashfs_compression])
         if not args.no_pxe:
-            cp_reflink(tmp_squashfs, os.path.join(tmpinitrd_pxe, 'root.squashfs'))
+            cp_reflink(tmp_squashfs, os.path.join(tmpinitrd_pxe_or_rootfs, 'root.squashfs'))
 
     # Generate initramfses
     # Add common content
@@ -222,8 +260,17 @@ def generate_iso():
         # Clone to PXE image
         pxe_initramfs = os.path.join(tmpdir, initramfs_img)
         cp_reflink(iso_initramfs, pxe_initramfs)
+        # Generate rootfs image
+        if is_live:
+            pxe_rootfs = os.path.join(tmpdir, 'rootfs.img')
+            extend_initrd(pxe_rootfs, tmpinitrd_pxe_rootfs)
+            # Save stream hash of rootfs for verifying out-of-band fetches
+            os.makedirs(os.path.join(tmpinitrd_pxe, 'etc'), exist_ok=True)
+            make_stream_hash(pxe_rootfs, os.path.join(tmpinitrd_pxe, 'etc/coreos-live-want-rootfs'))
         # Add additional content to PXE image
         extend_initrd(pxe_initramfs, tmpinitrd_pxe)
+    # Add additional content to ISO image
+    extend_initrd(iso_initramfs, tmpinitrd_iso)
     # Add Ignition padding to ISO image
     with open(iso_initramfs, 'ab') as fdst:
         fdst.write(bytes(initrd_ignition_padding))
@@ -467,6 +514,16 @@ def generate_iso():
                 'sha256': sha256sum_file(initramfs_file)
             }
         })
+        if is_live:
+            rootfs_name = f'{base_name}-{args.build}-{image_type}-rootfs.{basearch}.img'
+            rootfs_file = os.path.join(builddir, rootfs_name)
+            shutil.move(pxe_rootfs, rootfs_file)
+            buildmeta['images'].update({
+                'live-rootfs': {
+                    'path': rootfs_name,
+                    'sha256': sha256sum_file(rootfs_file)
+                }
+            })
 
     write_json(buildmeta_path, buildmeta)
     print(f"Updated: {buildmeta_path}")

src/cmd-compress

@@ -46,7 +46,7 @@ else:
 print(f"Targeting build: {build}")
 
 # Don't compress certain images
-imgs_to_skip = ["ostree", "iso", "rojig", "live-iso", "vmware", "initramfs", "live-initramfs", "kernel", "live-kernel"]
+imgs_to_skip = ["ostree", "iso", "rojig", "live-iso", "vmware", "initramfs", "live-initramfs", "kernel", "live-kernel", "live-rootfs"]
 
 
 def get_cpu_param(param):

src/schema/v1.json

@@ -337,6 +337,7 @@
        "live-kernel",
        "live-initramfs",
        "live-iso",
+       "live-rootfs",
        "metal",
        "metal4k",
        "openstack",
@@ -417,6 +418,12 @@
         "title":"Live ISO",
         "$ref": "#/definitions/artifact"
        },
+       "live-rootfs": {
+         "$id":"#/properties/images/properties/live-rootfs",
+         "type":"object",
+         "title":"Live Rootfs",
+         "$ref": "#/definitions/artifact"
+        },
        "openstack": {
          "$id":"#/properties/images/properties/openstack",
          "type":"object",