From 2d44a8f5261835126b9c56281ecac469a3e63e2c Mon Sep 17 00:00:00 2001 From: Moritz Sanft <58110325+msanft@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:51:06 +0100 Subject: [PATCH 1/2] packages/buildMicroVM: init This adds a builder for "MicroVM" images which are comprised of only a kernel, its kernel command line, and a root filesystem image. Additionally, this builder has a sanity check to see if the rootfs image contains an ESP, which indicates that an image thats not supposed to be a microVM was built with this builder. --- packages/by-name/buildMicroVM/package.nix | 36 +++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 packages/by-name/buildMicroVM/package.nix diff --git a/packages/by-name/buildMicroVM/package.nix b/packages/by-name/buildMicroVM/package.nix new file mode 100644 index 000000000..a4951805f --- /dev/null +++ b/packages/by-name/buildMicroVM/package.nix @@ -0,0 +1,36 @@ +# Copyright 2024 Edgeless Systems GmbH +# SPDX-License-Identifier: AGPL-3.0-only + +# Builds a micro VM image (i.e. rootfs, kernel and kernel cmdline) from a NixOS +# configuration. These components can then be booted in a microVM-fashion +# with QEMU's direct Linux boot feature. +# See: https://qemu-project.gitlab.io/qemu/system/linuxboot.html + +{ + symlinkJoin, + lib, + ... +}: + +nixos-config: + +lib.throwIf + (lib.foldlAttrs ( + acc: _: partConfig: + acc || (partConfig.repartConfig.Type == "esp") + ) false nixos-config.config.image.repart.partitions) + "MicroVM images should not contain an ESP." + + symlinkJoin + { + name = "microvm-image"; + + paths = [ + nixos-config.config.system.build.kernel + nixos-config.config.system.build.image + ]; + + postBuild = '' + echo -n ${lib.concatStringsSep " " nixos-config.config.boot.kernelParams} > $out/kernel-params + ''; + } From d618b701cf6fb4f7094dd7a1cd789f0ae1ba94c3 Mon Sep 17 00:00:00 2001 From: Moritz Sanft <58110325+msanft@users.noreply.github.com> Date: Tue, 12 Nov 2024 16:12:12 +0100 Subject: [PATCH 2/2] wip: we bootin! --- packages/by-name/boot-image/package.nix | 9 +++- packages/by-name/boot-microvm/package.nix | 38 ++++++++++++++ packages/by-name/buildMicroVM/package.nix | 16 ++++-- packages/by-name/image-podvm/package.nix | 6 +-- .../by-name/kata/kata-kernel-uvm/package.nix | 6 ++- .../by-name/kernel-podvm-qemu/package.nix | 51 +++++++++++++++++++ packages/by-name/mkNixosConfig/package.nix | 6 ++- packages/nixos/azure.nix | 2 + packages/nixos/image.nix | 28 +++++----- packages/nixos/qemu.nix | 27 ++++++++++ packages/nixos/system.nix | 1 - 11 files changed, 166 insertions(+), 24 deletions(-) create mode 100644 packages/by-name/boot-microvm/package.nix create mode 100644 packages/by-name/kernel-podvm-qemu/package.nix create mode 100644 packages/nixos/qemu.nix diff --git a/packages/by-name/boot-image/package.nix b/packages/by-name/boot-image/package.nix index 82ed3d191..16341d32b 100644 --- a/packages/by-name/boot-image/package.nix +++ b/packages/by-name/boot-image/package.nix @@ -8,9 +8,14 @@ }: writeShellApplication { - name = "boot-image"; + name = "boot-microvm"; runtimeInputs = [ qemu ]; text = '' + if [ ! -f "$1/kernel-params" ]; then + echo "Error: $1/kernel-params not found" >&2 + exit 1 + fi + tmpFile=$(mktemp) cp "$1" "$tmpFile" qemu-system-x86_64 \ @@ -19,6 +24,8 @@ writeShellApplication { -nographic \ -drive if=pflash,format=raw,readonly=on,file=${OVMF.firmware} \ -drive if=pflash,format=raw,readonly=on,file=${OVMF.variables} \ + -kernel $1/bzImage \ + -append "$(cat $1/kernel-params)" \ -drive "format=raw,file=$tmpFile" ''; } diff --git a/packages/by-name/boot-microvm/package.nix b/packages/by-name/boot-microvm/package.nix new file mode 100644 index 000000000..04f9eb128 --- /dev/null +++ b/packages/by-name/boot-microvm/package.nix @@ -0,0 +1,38 @@ +# Copyright 2024 Edgeless Systems GmbH +# SPDX-License-Identifier: AGPL-3.0-only + +{ + writeShellApplication, + qemu, + OVMF, +}: + +writeShellApplication { + name = "boot-image"; + runtimeInputs = [ qemu ]; + text = '' + if [ $# -ne 3 ]; then + echo "Usage: $0 " >&2 + exit 1 + fi + + kernel=$1 + # kernelParams=$(cat "$2") + image=$3 + + tmpFile=$(mktemp) + cp "$image" "$tmpFile" + + qemu-system-x86_64 \ + -enable-kvm \ + -m 3G \ + -nographic \ + -drive if=pflash,format=raw,readonly=on,file=${OVMF.firmware} \ + -drive if=pflash,format=raw,readonly=on,file=${OVMF.variables} \ + -kernel "$kernel" \ + -append "init=/nix/store/x45q9gkzj8wzw952lv2jrsyx8vqdfx1b-nixos-system-nixos-24.11pre-git/init root=/dev/sda1 rootfstype=erofs rootflags=ro console=ttyS0" \ + -device virtio-scsi-pci,id=scsi0,num_queues=4 \ + -device scsi-hd,drive=drive0,bus=scsi0.0,channel=0,scsi-id=0,lun=0 \ + -drive "file=$tmpFile,if=none,id=drive0" + ''; +} diff --git a/packages/by-name/buildMicroVM/package.nix b/packages/by-name/buildMicroVM/package.nix index a4951805f..173c75dc1 100644 --- a/packages/by-name/buildMicroVM/package.nix +++ b/packages/by-name/buildMicroVM/package.nix @@ -21,6 +21,10 @@ lib.throwIf ) false nixos-config.config.image.repart.partitions) "MicroVM images should not contain an ESP." + # lib.throwIf + # (lib.foldl' (acc: v: acc || (lib.hasInfix "root=" v)) false nixos-config.config.boot.kernelParams) + # "MicroVM images should not set the `root=` commandline parameter, as it will need to be decided by the VMM." + symlinkJoin { name = "microvm-image"; @@ -30,7 +34,13 @@ lib.throwIf nixos-config.config.system.build.image ]; - postBuild = '' - echo -n ${lib.concatStringsSep " " nixos-config.config.boot.kernelParams} > $out/kernel-params - ''; + postBuild = + let + kernelParams = nixos-config.config.boot.kernelParams ++ [ + "init=${nixos-config.config.system.build.toplevel}/init" + ]; + in + '' + echo -n ${lib.concatStringsSep " " kernelParams} > $out/kernel-params + ''; } diff --git a/packages/by-name/image-podvm/package.nix b/packages/by-name/image-podvm/package.nix index 26729aa10..d5d7a2563 100644 --- a/packages/by-name/image-podvm/package.nix +++ b/packages/by-name/image-podvm/package.nix @@ -2,7 +2,7 @@ # SPDX-License-Identifier: AGPL-3.0-only { - buildVerityUKI, + buildMicroVM, mkNixosConfig, withDebug ? true, @@ -10,10 +10,10 @@ withCSP ? "azure", }: -buildVerityUKI (mkNixosConfig { +buildMicroVM (mkNixosConfig { contrast = { debug.enable = withDebug; gpu.enable = withGPU; - azure.enable = withCSP == "azure"; + qemu.enable = true; }; }) diff --git a/packages/by-name/kata/kata-kernel-uvm/package.nix b/packages/by-name/kata/kata-kernel-uvm/package.nix index b3cc2cb02..4fa156d27 100644 --- a/packages/by-name/kata/kata-kernel-uvm/package.nix +++ b/packages/by-name/kata/kata-kernel-uvm/package.nix @@ -7,6 +7,7 @@ stdenvNoCC, fetchzip, kata, + ... # to satisfy `linuxPackagesFor`, which passes `features` to this. }: let @@ -28,7 +29,10 @@ let substituteInPlace $config \ --replace-fail 'CONFIG_INITRAMFS_SOURCE="initramfs.cpio.gz"' 'CONFIG_INITRAMFS_SOURCE=""' \ --replace-fail '# CONFIG_DM_INIT is not set' 'CONFIG_DM_INIT=y' \ - --replace-fail 'CONFIG_MODULE_SIG=y' 'CONFIG_MODULE_SIG=n' + --replace-fail 'CONFIG_MODULE_SIG=y' 'CONFIG_MODULE_SIG=n' \ + --replace-fail '# CONFIG_DMIID is not set' 'CONFIG_DMIID=y' \ + --replace-fail '# CONFIG_TMPFS_POSIX_ACL is not set' 'CONFIG_TMPFS_POSIX_ACL=y' \ + --replace-fail '# CONFIG_TMPFS_XATTR is not set' 'CONFIG_TMPFS_XATTR=y' ''; dontBuild = true; diff --git a/packages/by-name/kernel-podvm-qemu/package.nix b/packages/by-name/kernel-podvm-qemu/package.nix new file mode 100644 index 000000000..ccae43d5b --- /dev/null +++ b/packages/by-name/kernel-podvm-qemu/package.nix @@ -0,0 +1,51 @@ +# Copyright 2024 Edgeless Systems GmbH +# SPDX-License-Identifier: AGPL-3.0-only + +{ + lib, + fetchurl, + buildLinux, + ... +}: + +buildLinux { + version = "6.11"; + modDirVersion = "6.11.7"; + + src = fetchurl { + url = "https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-6.11.7.tar.xz"; + sha256 = "sha256-C/XsZEgX15KJIPdjWBMR9b8lipJ1nPLzCYXadDrz67I="; + }; + + structuredExtraConfig = with lib.kernel; { + AMD_MEM_ENCRYPT = lib.mkForce (option yes); + DRM_AMDGPU = lib.mkForce (option no); + DRM_AMDGPU_CIK = lib.mkForce (option no); + DRM_AMDGPU_SI = lib.mkForce (option no); + DRM_AMDGPU_USERPTR = lib.mkForce (option no); + DRM_AMD_DC_FP = lib.mkForce (option no); + DRM_AMD_DC_SI = lib.mkForce (option no); + HSA_AMD = lib.mkForce (option no); + DRM_AMD_ACP = lib.mkForce (option no); + DRM_AMD_DC_DCN = lib.mkForce (option no); + DRM_AMD_DC_HDCP = lib.mkForce (option no); + DRM_AMD_SECURE_DISPLAY = lib.mkForce (option no); + DRM_AMD_ISP = lib.mkForce (option no); + HYPERV_AZURE_BLOB = lib.mkForce (option no); + INTEL_TDX_GUEST = lib.mkForce (option yes); + DEFAULT_SECURITY_APPARMOR = lib.mkForce (option no); + DEFAULT_SECURITY_SELINUX = lib.mkForce (option no); + + # Required to be compiled with the kernel to allow booting in + # direct Linux boot scenarios. + VIRTIO = lib.mkForce (option yes); + VIRTIO_PCI = lib.mkForce (option yes); + VIRTIO_BLK = lib.mkForce (option yes); + VIRTIO_SCSI = lib.mkForce (option yes); + VIRTIO_MMIO = lib.mkForce (option yes); + ATA = lib.mkForce (option yes); + EROFS_FS = lib.mkForce (option yes); + }; + + extraMeta.branch = "6.11"; +} diff --git a/packages/by-name/mkNixosConfig/package.nix b/packages/by-name/mkNixosConfig/package.nix index 3b08d93d5..eae2f909c 100644 --- a/packages/by-name/mkNixosConfig/package.nix +++ b/packages/by-name/mkNixosConfig/package.nix @@ -40,11 +40,15 @@ lib.makeOverridable ( azure-no-agent cloud-api-adaptor kernel-podvm-azure + kernel-podvm-qemu pause-bundle nvidia-ctk-oci-hook nvidia-ctk-with-config ; - inherit (outerPkgs.kata) kata-agent; + inherit (outerPkgs.kata) + kata-agent + kata-kernel-uvm + ; }) ]; diff --git a/packages/nixos/azure.nix b/packages/nixos/azure.nix index 612b48e76..ec1e3a4c2 100644 --- a/packages/nixos/azure.nix +++ b/packages/nixos/azure.nix @@ -55,6 +55,8 @@ in }; config = lib.mkIf cfg.enable { + boot.kernelPackages = pkgs.recurseIntoAttrs (pkgs.linuxPackagesFor pkgs.kernel-podvm-azure); + boot.initrd = { kernelModules = [ "hv_storvsc" diff --git a/packages/nixos/image.nix b/packages/nixos/image.nix index 192102e5c..2c4448d7c 100644 --- a/packages/nixos/image.nix +++ b/packages/nixos/image.nix @@ -15,20 +15,20 @@ # This defines the actual partition layout. partitions = { # EFI System Partition, holds the UKI. - "00-esp" = { - contents = { - "/".source = pkgs.runCommand "esp-contents" { } '' - mkdir -p $out/EFI/BOOT - cp ${config.system.build.uki}/${config.system.boot.loader.ukiFile} $out/EFI/BOOT/BOOTX64.EFI - ''; - }; - repartConfig = { - Type = "esp"; - Format = "vfat"; - SizeMinBytes = "64M"; - UUID = "null"; # Fix partition UUID for reproducibility. - }; - }; + # "00-esp" = { + # contents = { + # "/".source = pkgs.runCommand "esp-contents" { } '' + # mkdir -p $out/EFI/BOOT + # cp ${config.system.build.uki}/${config.system.boot.loader.ukiFile} $out/EFI/BOOT/BOOTX64.EFI + # ''; + # }; + # repartConfig = { + # Type = "esp"; + # Format = "vfat"; + # SizeMinBytes = "64M"; + # UUID = "null"; # Fix partition UUID for reproducibility. + # }; + # }; # Root filesystem. "10-root" = { diff --git a/packages/nixos/qemu.nix b/packages/nixos/qemu.nix new file mode 100644 index 000000000..88a3af560 --- /dev/null +++ b/packages/nixos/qemu.nix @@ -0,0 +1,27 @@ +# Copyright 2024 Edgeless Systems GmbH +# SPDX-License-Identifier: AGPL-3.0-only + +{ + config, + pkgs, + lib, + ... +}: + +let + cfg = config.contrast.qemu; +in + +{ + options.contrast.qemu = { + enable = lib.mkEnableOption "Enable QEMU (bare-metal) specific settings"; + }; + + config = lib.mkIf cfg.enable { + boot.kernelPackages = pkgs.recurseIntoAttrs (pkgs.linuxPackagesFor pkgs.kata-kernel-uvm); + + boot.initrd.systemd.tpm2.enable = lib.mkForce false; + boot.initrd.systemd.enable = lib.mkForce false; + boot.initrd.availableKernelModules = lib.mkForce [ ]; + }; +} diff --git a/packages/nixos/system.nix b/packages/nixos/system.nix index d11a336ac..d973603c5 100644 --- a/packages/nixos/system.nix +++ b/packages/nixos/system.nix @@ -10,7 +10,6 @@ { boot.loader.grub.enable = false; - boot.kernelPackages = pkgs.recurseIntoAttrs (pkgs.linuxPackagesFor pkgs.kernel-podvm-azure); boot.kernelParams = [ "systemd.verity=yes" "selinux=0"