From 1aec2a73baf2a4eb94fe670f22e5d13ad06d398b Mon Sep 17 00:00:00 2001 From: Moritz Sanft <58110325+msanft@users.noreply.github.com> Date: Thu, 19 Dec 2024 16:19:41 +0100 Subject: [PATCH 1/2] node-installer: pass through CDI annotations For GPU support, Kata needs CDI pod annotations. We need to configure containerd to pass these through to Kata. --- nodeinstaller/internal/constants/constants.go | 4 + nodeinstaller/node-installer_test.go | 4 +- .../expected-bare-metal-qemu-snp-gpu.toml | 81 +++++++++++++++++++ 3 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 nodeinstaller/testdata/expected-bare-metal-qemu-snp-gpu.toml diff --git a/nodeinstaller/internal/constants/constants.go b/nodeinstaller/internal/constants/constants.go index de5d2dc751..db0e05db34 100644 --- a/nodeinstaller/internal/constants/constants.go +++ b/nodeinstaller/internal/constants/constants.go @@ -137,6 +137,10 @@ func ContainerdRuntimeConfigFragment(baseDir, snapshotter string, platform platf cfg.Options = map[string]any{ "ConfigPath": filepath.Join(baseDir, "etc", "configuration-qemu-snp.toml"), } + // For GPU support, we need to pass through the CDI annotations. + if platform == platforms.K3sQEMUSNPGPU { + cfg.PodAnnotations = append(cfg.PodAnnotations, "cdi.k8s.io/*") + } default: return nil, fmt.Errorf("unsupported platform: %s", platform) } diff --git a/nodeinstaller/node-installer_test.go b/nodeinstaller/node-installer_test.go index a1d14a4fc4..526a99d79f 100644 --- a/nodeinstaller/node-installer_test.go +++ b/nodeinstaller/node-installer_test.go @@ -22,6 +22,8 @@ var ( expectedConfBareMetalQEMUTDX []byte //go:embed testdata/expected-bare-metal-qemu-snp.toml expectedConfBareMetalQEMUSNP []byte + //go:embed testdata/expected-bare-metal-qemu-snp-gpu.toml + expectedConfBareMetalQEMUSNPGPU []byte ) func TestPatchContainerdConfig(t *testing.T) { @@ -44,7 +46,7 @@ func TestPatchContainerdConfig(t *testing.T) { }, "BareMetalQEMUSNPGPU": { platform: platforms.K3sQEMUSNPGPU, - expected: expectedConfBareMetalQEMUSNP, + expected: expectedConfBareMetalQEMUSNPGPU, }, "Unknown": { platform: platforms.Unknown, diff --git a/nodeinstaller/testdata/expected-bare-metal-qemu-snp-gpu.toml b/nodeinstaller/testdata/expected-bare-metal-qemu-snp-gpu.toml new file mode 100644 index 0000000000..3cf6bb66e2 --- /dev/null +++ b/nodeinstaller/testdata/expected-bare-metal-qemu-snp-gpu.toml @@ -0,0 +1,81 @@ +version = 2 + +[debug] +level = 'debug' + +[metrics] +address = '0.0.0.0:10257' + +[plugins] +[plugins.'io.containerd.grpc.v1.cri'] +sandbox_image = 'mcr.microsoft.com/oss/kubernetes/pause:3.6' + +[plugins.'io.containerd.grpc.v1.cri'.cni] +bin_dir = '/opt/cni/bin' +conf_dir = '/etc/cni/net.d' +conf_template = '/etc/containerd/kubenet_template.conf' + +[plugins.'io.containerd.grpc.v1.cri'.containerd] +default_runtime_name = 'runc' +disable_snapshot_annotations = false +discard_unpacked_layers = false + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes] +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.kata] +runtime_type = 'io.containerd.kata.v2' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.kata-cc] +pod_annotations = ['io.katacontainers.*'] +privileged_without_host_devices = true +runtime_type = 'io.containerd.kata-cc.v2' +snapshotter = 'tardev' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.kata-cc.options] +ConfigPath = '/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.katacli] +runtime_type = 'io.containerd.runc.v1' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.katacli.options] +BinaryName = '/usr/bin/kata-runtime' +CriuPath = '' +IoGid = 0 +IoUid = 0 +NoNewKeyring = false +NoPivotRoot = false +Root = '' +ShimCgroup = '' +SystemdCgroup = false + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.my-runtime] +runtime_type = 'io.containerd.contrast-cc.v2' +runtime_path = '/opt/edgeless/my-runtime/bin/containerd-shim-contrast-cc-v2' +pod_annotations = ['io.katacontainers.*', 'cdi.k8s.io/*'] +privileged_without_host_devices = true +snapshotter = 'nydus-my-runtime' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.my-runtime.options] +ConfigPath = '/opt/edgeless/my-runtime/etc/configuration-qemu-snp.toml' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.runc] +runtime_type = 'io.containerd.runc.v2' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.runc.options] +BinaryName = '/usr/bin/runc' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.untrusted] +runtime_type = 'io.containerd.runc.v2' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.untrusted.options] +BinaryName = '/usr/bin/runc' + +[plugins.'io.containerd.grpc.v1.cri'.registry] +config_path = '/etc/containerd/certs.d' + +[plugins.'io.containerd.grpc.v1.cri'.registry.headers] +X-Meta-Source-Client = ['azure/aks'] + +[proxy_plugins] +[proxy_plugins.nydus-my-runtime] +type = 'snapshot' +address = '/run/containerd/containerd-nydus-grpc-my-runtime.sock' From 5305580d01bbb4f24249b7101dc7d3d2fcaab5b5 Mon Sep 17 00:00:00 2001 From: Moritz Sanft <58110325+msanft@users.noreply.github.com> Date: Thu, 19 Dec 2024 16:21:18 +0100 Subject: [PATCH 2/2] node-installer: add GPU-specific options --- nodeinstaller/internal/constants/constants.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nodeinstaller/internal/constants/constants.go b/nodeinstaller/internal/constants/constants.go index db0e05db34..a245b8f43a 100644 --- a/nodeinstaller/internal/constants/constants.go +++ b/nodeinstaller/internal/constants/constants.go @@ -94,6 +94,14 @@ func KataRuntimeConfig(baseDir string, platform platforms.Platform, qemuExtraKer if debug { config.Hypervisor["qemu"]["enable_debug"] = true } + // GPU-specific settings + if platform == platforms.K3sQEMUSNPGPU { + config.Hypervisor["qemu"]["guest_hook_path"] = "/usr/share/oci/hooks" + config.Hypervisor["qemu"]["cold_plug_vfio"] = "root-port" + // GPU images tend to be larger, so give a better default timeout that + // allows for pulling those. + config.Runtime["create_container_timeout"] = 600 + } default: return nil, fmt.Errorf("unsupported platform: %s", platform) }