Skip to content

Commit

Permalink
node-installer: install TDX and SNP specific components
Browse files Browse the repository at this point in the history 
The node-installer now uploads distinct QEMU and OVMF blobs for TDX and SNP, as both need separate patches. The QEMU for TDX is taken from Ubuntu upstream repositories for now, until we can build a static QEMU with their patches.
  • Loading branch information
@msanft
msanft committed Jul 18, 2024
1 parent 8bb574c commit 288165e
Show file tree
Hide file tree
Showing 4 changed files with 146 additions and 65 deletions.
12 changes: 6 additions & 6 deletions node-installer/internal/constants/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,11 @@ func KataRuntimeConfig(baseDir string, platform platforms.Platform, debug bool)
if err := toml.Unmarshal([]byte(kataBareMetalQEMUTDXBaseConfig), &config); err != nil {
return nil, fmt.Errorf("failed to unmarshal kata runtime configuration: %w", err)
}
config.Hypervisor["qemu"]["path"] = filepath.Join(baseDir, "bin", "qemu-system-x86_64")
config.Hypervisor["qemu"]["firmware"] = filepath.Join(baseDir, "share", "OVMF.fd")
config.Hypervisor["qemu"]["path"] = filepath.Join(baseDir, "tdx", "bin", "qemu-system-x86_64")
config.Hypervisor["qemu"]["firmware"] = filepath.Join(baseDir, "tdx", "share", "OVMF.fd")
config.Hypervisor["qemu"]["image"] = filepath.Join(baseDir, "share", "kata-containers.img")
config.Hypervisor["qemu"]["kernel"] = filepath.Join(baseDir, "share", "kata-kernel")
config.Hypervisor["qemu"]["valid_hypervisor_paths"] = []string{filepath.Join(baseDir, "bin", "qemu-system-x86_64")}
config.Hypervisor["qemu"]["valid_hypervisor_paths"] = []string{filepath.Join(baseDir, "tdx", "bin", "qemu-system-x86_64")}
if debug {
config.Hypervisor["qemu"]["enable_debug"] = true
config.Hypervisor["qemu"]["kernel_params"] = " agent.log=debug initcall_debug"
Expand All @@ -76,14 +76,14 @@ func KataRuntimeConfig(baseDir string, platform platforms.Platform, debug bool)
if err := toml.Unmarshal([]byte(kataBareMetalQEMUSNPBaseConfig), &config); err != nil {
return nil, fmt.Errorf("failed to unmarshal kata runtime configuration: %w", err)
}
config.Hypervisor["qemu"]["path"] = filepath.Join(baseDir, "bin", "qemu-system-x86_64")
config.Hypervisor["qemu"]["firmware"] = filepath.Join(baseDir, "share", "OVMF.fd")
config.Hypervisor["qemu"]["path"] = filepath.Join(baseDir, "snp", "bin", "qemu-system-x86_64")
config.Hypervisor["qemu"]["firmware"] = filepath.Join(baseDir, "snp", "share", "OVMF.fd")
config.Hypervisor["qemu"]["image"] = filepath.Join(baseDir, "share", "kata-containers.img")
config.Hypervisor["qemu"]["kernel"] = filepath.Join(baseDir, "share", "kata-kernel")
delete(config.Hypervisor["qemu"], "initrd")
config.Hypervisor["qemu"]["block_device_aio"] = "threads"
config.Hypervisor["qemu"]["shared_fs"] = "virtio-9p"
config.Hypervisor["qemu"]["valid_hypervisor_paths"] = []string{filepath.Join(baseDir, "bin", "qemu-system-x86_64")}
config.Hypervisor["qemu"]["valid_hypervisor_paths"] = []string{filepath.Join(baseDir, "snp", "bin", "qemu-system-x86_64")}
config.Hypervisor["qemu"]["rootfs_type"] = "erofs"
if debug {
config.Hypervisor["qemu"]["enable_debug"] = true
Expand Down
62 changes: 34 additions & 28 deletions node-installer/node-installer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,26 +63,18 @@ func run(ctx context.Context, fetcher assetFetcher, platform platforms.Platform,
if err := config.Validate(); err != nil {
return fmt.Errorf("validating config: %w", err)
}
fmt.Printf("Using config: %+v\n", config)

runtimeBase := filepath.Join("/opt", "edgeless", config.RuntimeHandlerName)
binDir := filepath.Join(hostMount, runtimeBase, "bin")

// Create directory structure
if err := os.MkdirAll(binDir, os.ModePerm); err != nil {
return fmt.Errorf("creating runtime bin directory: %w", err)
}
if err := os.MkdirAll(filepath.Join(hostMount, runtimeBase, "share/qemu"), os.ModePerm); err != nil {
return fmt.Errorf("creating runtime share directory: %w", err)
}
if err := os.MkdirAll(filepath.Join(hostMount, runtimeBase, "etc"), os.ModePerm); err != nil {
return fmt.Errorf("creating runtime etc directory: %w", err)
}
if err := os.MkdirAll(filepath.Join(hostMount, "etc", "containerd"), os.ModePerm); err != nil {
return fmt.Errorf("creating /etc/containerd directory: %w", err)
}

// Copy the files
for _, file := range config.Files {
fmt.Printf("Fetching %q to %q\n", file.URL, file.Path)

if err := os.MkdirAll(filepath.Dir(filepath.Join(hostMount, file.Path)), os.ModePerm); err != nil {
return fmt.Errorf("creating directory %q: %w", filepath.Dir(file.Path), err)
}

var fetchErr error
if file.Integrity == "" {
_, fetchErr = fetcher.FetchUnchecked(ctx, file.URL, filepath.Join(hostMount, file.Path))
Expand All @@ -93,35 +85,47 @@ func run(ctx context.Context, fetcher assetFetcher, platform platforms.Platform,
return fmt.Errorf("fetching file from %q to %q: %w", file.URL, file.Path, fetchErr)
}
}
items, err := os.ReadDir(binDir)
if err != nil {
return fmt.Errorf("reading bin directory %q: %w", binDir, err)
}

for _, item := range items {
if !item.Type().IsRegular() {
continue
// Fix-up the permissions of executables
binDirs := []string{
filepath.Join(hostMount, runtimeBase, "bin"),
filepath.Join(hostMount, runtimeBase, "tdx", "bin"),
filepath.Join(hostMount, runtimeBase, "snp", "bin"),
}
for _, binDir := range binDirs {
items, err := os.ReadDir(binDir)
if err != nil {
return fmt.Errorf("reading bin directory %q: %w", binDir, err)
}
if err := os.Chmod(filepath.Join(binDir, item.Name()), 0o755); err != nil {
return fmt.Errorf("chmod %q: %w", item.Name(), err)

for _, item := range items {
if !item.Type().IsRegular() {
continue
}
if err := os.Chmod(filepath.Join(binDir, item.Name()), 0o755); err != nil {
return fmt.Errorf("chmod %q: %w", item.Name(), err)
}
}
}

kataConfigPath := filepath.Join(hostMount, runtimeBase, "etc")
if err := os.MkdirAll(kataConfigPath, os.ModePerm); err != nil {
return fmt.Errorf("creating directory %q: %w", kataConfigPath, err)
}
var containerdConfigPath string
switch platform {
case platforms.AKSCloudHypervisorSNP:
kataConfigPath = filepath.Join(kataConfigPath, "configuration-clh-snp.toml")
containerdConfigPath = filepath.Join(hostMount, "etc", "containerd", "config.toml")
case platforms.K3sQEMUTDX:
kataConfigPath = filepath.Join(kataConfigPath, "configuration-qemu-tdx.toml")
containerdConfigPath = filepath.Join(hostMount, "var", "lib", "rancher", "k3s", "agent", "etc", "containerd", "config.toml")
case platforms.K3sQEMUSNP:
kataConfigPath = filepath.Join(kataConfigPath, "configuration-qemu-snp.toml")
containerdConfigPath = filepath.Join(hostMount, "var", "lib", "rancher", "k3s", "agent", "etc", "containerd", "config.toml.tmpl")
case platforms.K3sQEMUTDX:
kataConfigPath = filepath.Join(kataConfigPath, "configuration-qemu-tdx.toml")
containerdConfigPath = filepath.Join(hostMount, "var", "lib", "rancher", "k3s", "agent", "etc", "containerd", "config.toml.tmpl")
case platforms.RKE2QEMUTDX:
kataConfigPath = filepath.Join(kataConfigPath, "configuration-qemu-tdx.toml")
containerdConfigPath = filepath.Join(hostMount, "var", "lib", "rancher", "rke2", "agent", "etc", "containerd", "config.toml")
containerdConfigPath = filepath.Join(hostMount, "var", "lib", "rancher", "rke2", "agent", "etc", "containerd", "config.toml.tmpl")
default:
return fmt.Errorf("unsupported platform %q", platform)
}
Expand Down Expand Up @@ -245,6 +249,7 @@ func patchContainerdConfigTemplate(runtimeName, basePath, configTemplatePath str
if err != nil {
return fmt.Errorf("reading containerd config template: %w", err)
}
fmt.Printf("Existing containerd config template:\n%s\n", existingConfig)

// Don't add the runtime section if it already exists.
runtimeSection := fmt.Sprintf("[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.%s]", runtimeName)
Expand Down Expand Up @@ -329,6 +334,7 @@ func patchContainerdConfigTemplate(runtimeName, basePath, configTemplatePath str
newRawConfig = append(newRawConfig, []byte("\n")...)
}

fmt.Printf("New containerd config template:\n%s\n", newRawConfig)
return os.WriteFile(configTemplatePath, newRawConfig, os.ModePerm)
}

Expand Down
100 changes: 76 additions & 24 deletions packages/by-name/kata/contrast-node-installer-image/package.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,20 @@ let
path = "/opt/edgeless/${runtime-handler}/share/kata-kernel";
}
{
url = "file:///opt/edgeless/bin/qemu-system-x86_64";
path = "/opt/edgeless/${runtime-handler}/bin/qemu-system-x86_64";
url = "file:///opt/edgeless/snp/bin/qemu-system-x86_64";
path = "/opt/edgeless/${runtime-handler}/snp/bin/qemu-system-x86_64";
}
{
url = "file:///opt/edgeless/share/OVMF.fd";
path = "/opt/edgeless/${runtime-handler}/share/OVMF.fd";
url = "file:///opt/edgeless/tdx/bin/qemu-system-x86_64";
path = "/opt/edgeless/${runtime-handler}/tdx/bin/qemu-system-x86_64";
}
{
url = "file:///opt/edgeless/snp/share/OVMF.fd";
path = "/opt/edgeless/${runtime-handler}/snp/share/OVMF.fd";
}
{
url = "file:///opt/edgeless/tdx/share/OVMF.fd";
path = "/opt/edgeless/${runtime-handler}/tdx/share/OVMF.fd";
}
{
url = "file:///opt/edgeless/bin/containerd-shim-contrast-cc-v2";
Expand All @@ -63,16 +71,28 @@ let
path = "/opt/edgeless/${runtime-handler}/bin/kata-runtime";
}
{
url = "file:///opt/edgeless/share/qemu/kvmvapic.bin";
path = "/opt/edgeless/${runtime-handler}/share/qemu/kvmvapic.bin";
url = "file:///opt/edgeless/snp/share/qemu/kvmvapic.bin";
path = "/opt/edgeless/${runtime-handler}/snp/share/qemu/kvmvapic.bin";
}
{
url = "file:///opt/edgeless/snp/share/qemu/linuxboot_dma.bin";
path = "/opt/edgeless/${runtime-handler}/snp/share/qemu/linuxboot_dma.bin";
}
{
url = "file:///opt/edgeless/snp/share/qemu/efi-virtio.rom";
path = "/opt/edgeless/${runtime-handler}/snp/share/qemu/efi-virtio.rom";
}
{
url = "file:///opt/edgeless/tdx/share/qemu/kvmvapic.bin";
path = "/opt/edgeless/${runtime-handler}/tdx/share/qemu/kvmvapic.bin";
}
{
url = "file:///opt/edgeless/share/qemu/linuxboot_dma.bin";
path = "/opt/edgeless/${runtime-handler}/share/qemu/linuxboot_dma.bin";
url = "file:///opt/edgeless/tdx/share/qemu/linuxboot_dma.bin";
path = "/opt/edgeless/${runtime-handler}/tdx/share/qemu/linuxboot_dma.bin";
}
{
url = "file:///opt/edgeless/share/qemu/efi-virtio.rom";
path = "/opt/edgeless/${runtime-handler}/share/qemu/efi-virtio.rom";
url = "file:///opt/edgeless/tdx/share/qemu/efi-virtio.rom";
path = "/opt/edgeless/${runtime-handler}/tdx/share/qemu/efi-virtio.rom";
}
];
runtimeHandlerName = runtime-handler;
Expand All @@ -96,32 +116,62 @@ let
];
};

ovmf = ociLayerTar {
ovmf-snp = ociLayerTar {
files = [
{
source = kata.runtime-class-files.ovmf-snp;
destination = "/opt/edgeless/snp/share/OVMF.fd";
}
];
};

qemu-snp = ociLayerTar {
files = [
{
source = kata.runtime-class-files.qemu-snp.bin;
destination = "/opt/edgeless/snp/bin/qemu-system-x86_64";
}
{
source = "${kata.runtime-class-files.qemu-snp.share}/kvmvapic.bin";
destination = "/opt/edgeless/snp/share/qemu/kvmvapic.bin";
}
{
source = "${kata.runtime-class-files.qemu-snp.share}/linuxboot_dma.bin";
destination = "/opt/edgeless/snp/share/qemu/linuxboot_dma.bin";
}
{
source = "${kata.runtime-class-files.qemu-snp.share}/efi-virtio.rom";
destination = "/opt/edgeless/snp/share/qemu/efi-virtio.rom";
}
];
};

ovmf-tdx = ociLayerTar {
files = [
{
source = kata.runtime-class-files.ovmf;
destination = "/opt/edgeless/share/OVMF.fd";
source = kata.runtime-class-files.ovmf-tdx;
destination = "/opt/edgeless/tdx/share/OVMF.fd";
}
];
};

qemu = ociLayerTar {
qemu-tdx = ociLayerTar {
files = [
{
source = kata.runtime-class-files.qemu-bin;
destination = "/opt/edgeless/bin/qemu-system-x86_64";
source = kata.runtime-class-files.qemu-tdx.bin;
destination = "/opt/edgeless/tdx/bin/qemu-system-x86_64";
}
{
source = "${kata.runtime-class-files.qemu-share}/kvmvapic.bin";
destination = "/opt/edgeless/share/qemu/kvmvapic.bin";
source = "${kata.runtime-class-files.qemu-tdx.share}/kvmvapic.bin";
destination = "/opt/edgeless/tdx/share/qemu/kvmvapic.bin";
}
{
source = "${kata.runtime-class-files.qemu-share}/linuxboot_dma.bin";
destination = "/opt/edgeless/share/qemu/linuxboot_dma.bin";
source = "${kata.runtime-class-files.qemu-tdx.share}/linuxboot_dma.bin";
destination = "/opt/edgeless/tdx/share/qemu/linuxboot_dma.bin";
}
{
source = "${kata.runtime-class-files.qemu-share}/efi-virtio.rom";
destination = "/opt/edgeless/share/qemu/efi-virtio.rom";
source = "${kata.runtime-class-files.qemu-tdx.share}/efi_virtio.rom";
destination = "/opt/edgeless/tdx/share/qemu/efi-virtio.rom";
}
];
};
Expand All @@ -144,8 +194,10 @@ let
node-installer
installer-config
kata-container-img
ovmf
qemu
ovmf-snp
ovmf-tdx
qemu-snp
qemu-tdx
kata-runtime
];
extraConfig = {
Expand Down
37 changes: 30 additions & 7 deletions packages/by-name/kata/runtime-class-files/package.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,40 @@
stdenvNoCC,
kata,
OVMF-SNP,
OVMF,
debugRuntime ? false,
qemu-static,
fetchzip,
}:

let
image = kata.kata-image;
kernel = "${kata.kata-kernel-uvm}/bzImage";

qemu-bin = "${qemu-static}/bin/qemu-system-x86_64";
qemu-share = "${qemu-static}/share/qemu";
qemu-snp = {
bin = "${qemu-static}/bin/qemu-system-x86_64";
share = "${qemu-static}/share/qemu";
};

ovmf-snp = "${OVMF-SNP}/FV/OVMF.fd";

# TODO(msanft): Incorporate the Canonical TDX QEMU patches in our QEMU build for a dynamically
# built SEV / TDX QEMU binary. For now, take the blob from a build of the following, which matches
# what Canonical provides in Ubuntu 24.04.
# https://code.launchpad.net/~kobuk-team/+recipe/tdx-qemu-noble
qemu-tdx =
let
qemu-tdx-blob = fetchzip {
url = "https://cdn.confidential.cloud/contrast/node-components/1%3A8.2.2%2Bds-0ubuntu2%2Btdx1.0~tdx1.202407031834~ubuntu24.04.1/1%3A8.2.2%2Bds-0ubuntu2%2Btdx1.0~tdx1.202407031834~ubuntu24.04.1.zip";
hash = "sha256-6TztmmmO2N1jk/cNKdvd/MMIf43N7lxPaasjKARRVik=";
};
in
{
bin = "${qemu-tdx-blob}/bin/qemu-system-x86_64";
share = "${qemu-tdx-blob}/share/qemu";
};

ovmf = "${OVMF-SNP}/FV/OVMF.fd";
ovmf-tdx = "${OVMF.fd}/FV/OVMF.fd";

containerd-shim-contrast-cc-v2 = "${kata.kata-runtime}/bin/containerd-shim-kata-v2";

Expand All @@ -32,18 +54,19 @@ stdenvNoCC.mkDerivation {
# TODO(msanft): perform the actual launch digest calculation.
buildPhase = ''
mkdir -p $out
sha256sum ${image} ${kernel} ${qemu-bin} ${qemu-share}/kvmvapic.bin ${qemu-share}/linuxboot_dma.bin ${qemu-share}/efi-virtio.rom ${containerd-shim-contrast-cc-v2} ${ovmf} | sha256sum | cut -d " " -f 1 > $out/launch-digest.hex
sha256sum ${image} ${kernel} ${qemu-snp.bin} ${qemu-tdx.bin} ${containerd-shim-contrast-cc-v2} ${ovmf-snp} ${ovmf-tdx} | sha256sum | cut -d " " -f 1 > $out/launch-digest.hex
printf "contrast-cc-%s" "$(cat $out/launch-digest.hex | head -c 32)" > $out/runtime-handler
'';

passthru = {
inherit
kernel
image
qemu-bin
qemu-share
qemu-tdx
qemu-snp
containerd-shim-contrast-cc-v2
ovmf
ovmf-tdx
ovmf-snp
kata-runtime
debugRuntime
;
Expand Down

0 comments on commit 288165e

Please sign in to comment.