diff --git a/.github/workflows/e2e_manual.yml b/.github/workflows/e2e_manual.yml index 13beda09c..be65db14d 100644 --- a/.github/workflows/e2e_manual.yml +++ b/.github/workflows/e2e_manual.yml @@ -46,7 +46,7 @@ jobs: echo "runner=ubuntu-22.04" >> "$GITHUB_OUTPUT" echo "self-hosted=false" >> "$GITHUB_OUTPUT" ;; - "K3s-QEMU-SNP") + "K3s-QEMU-SNP"|"K3s-QEMU-SNP-GPU") echo "runner=SNP" >> "$GITHUB_OUTPUT" echo "self-hosted=true" >> "$GITHUB_OUTPUT" ;; diff --git a/cli/genpolicy/config.go b/cli/genpolicy/config.go index 5b479580e..327f520f6 100644 --- a/cli/genpolicy/config.go +++ b/cli/genpolicy/config.go @@ -43,7 +43,7 @@ func NewConfig(platform platforms.Platform) *Config { Settings: aksSettings, Bin: aksGenpolicyBin, } - case platforms.MetalQEMUSNP, platforms.MetalQEMUTDX, platforms.K3sQEMUSNP, platforms.K3sQEMUTDX, platforms.RKE2QEMUTDX: + case platforms.MetalQEMUSNP, platforms.MetalQEMUTDX, platforms.K3sQEMUSNP, platforms.K3sQEMUSNPGPU, platforms.K3sQEMUTDX, platforms.RKE2QEMUTDX: return &Config{ Rules: kataRules, Settings: kataSettings, diff --git a/cli/main.go b/cli/main.go index 63cc5671b..e2a2c6183 100644 --- a/cli/main.go +++ b/cli/main.go @@ -105,7 +105,7 @@ func buildVersionString() (string, error) { switch platform { case platforms.AKSCloudHypervisorSNP: fmt.Fprintf(versionsWriter, "\tgenpolicy version:\t%s\n", constants.MicrosoftGenpolicyVersion) - case platforms.MetalQEMUSNP, platforms.MetalQEMUTDX, platforms.K3sQEMUSNP, platforms.K3sQEMUTDX, platforms.RKE2QEMUTDX: + case platforms.MetalQEMUSNP, platforms.MetalQEMUTDX, platforms.K3sQEMUSNP, platforms.K3sQEMUTDX, platforms.K3sQEMUSNPGPU, platforms.RKE2QEMUTDX: fmt.Fprintf(versionsWriter, "\tgenpolicy version:\t%s\n", constants.KataGenpolicyVersion) } } diff --git a/e2e/internal/contrasttest/contrasttest.go b/e2e/internal/contrasttest/contrasttest.go index 96e13e37d..7ed77bc29 100644 --- a/e2e/internal/contrasttest/contrasttest.go +++ b/e2e/internal/contrasttest/contrasttest.go @@ -202,7 +202,7 @@ func (ct *ContrastTest) patchReferenceValues(t *testing.T, platform platforms.Pl SNPVersion: toPtr(manifest.SVN(255)), MicrocodeVersion: toPtr(manifest.SVN(255)), } - case platforms.MetalQEMUSNP, platforms.K3sQEMUSNP: + case platforms.MetalQEMUSNP, platforms.K3sQEMUSNP, platforms.K3sQEMUSNPGPU: // The generate command doesn't fill in all required fields when // generating a manifest for baremetal SNP. Do that now. for i, snp := range m.ReferenceValues.SNP { @@ -372,7 +372,7 @@ func (ct *ContrastTest) FactorPlatformTimeout(timeout time.Duration) time.Durati switch ct.Platform { case platforms.AKSCloudHypervisorSNP: // AKS defined is the baseline return timeout - case platforms.MetalQEMUSNP, platforms.MetalQEMUTDX, platforms.K3sQEMUSNP, platforms.K3sQEMUTDX, platforms.RKE2QEMUTDX: + case platforms.MetalQEMUSNP, platforms.MetalQEMUTDX, platforms.K3sQEMUSNP, platforms.K3sQEMUSNPGPU, platforms.K3sQEMUTDX, platforms.RKE2QEMUTDX: return 2 * timeout default: return timeout diff --git a/internal/kuberesource/parts.go b/internal/kuberesource/parts.go index 7ec6e40bd..a47754263 100644 --- a/internal/kuberesource/parts.go +++ b/internal/kuberesource/parts.go @@ -136,7 +136,7 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle WithType(corev1.HostPathDirectory), )) snapshotterVolumes = nydusSnapshotterVolumes - case platforms.K3sQEMUTDX, platforms.K3sQEMUSNP, platforms.RKE2QEMUTDX: + case platforms.K3sQEMUTDX, platforms.K3sQEMUSNP, platforms.K3sQEMUSNPGPU, platforms.RKE2QEMUTDX: nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata:latest" snapshotter = nydusSnapshotter nydusSnapshotterVolumes = append(nydusSnapshotterVolumes, Volume(). diff --git a/internal/platforms/platforms.go b/internal/platforms/platforms.go index 106f6f733..3c73f20e8 100644 --- a/internal/platforms/platforms.go +++ b/internal/platforms/platforms.go @@ -28,11 +28,13 @@ const ( MetalQEMUSNP // MetalQEMUTDX is the generic platform for bare-metal TDX deployments. MetalQEMUTDX + // K3sQEMUSNPGPU represents a deployment with QEMU on bare-metal SNP K3s with GPU passthrough. + K3sQEMUSNPGPU ) // All returns a list of all available platforms. func All() []Platform { - return []Platform{AKSCloudHypervisorSNP, K3sQEMUTDX, K3sQEMUSNP, RKE2QEMUTDX, MetalQEMUSNP, MetalQEMUTDX} + return []Platform{AKSCloudHypervisorSNP, K3sQEMUTDX, K3sQEMUSNP, RKE2QEMUTDX, MetalQEMUSNP, MetalQEMUTDX, K3sQEMUSNPGPU} } // AllStrings returns a list of all available platforms as strings. @@ -53,6 +55,8 @@ func (p Platform) String() string { return "K3s-QEMU-TDX" case K3sQEMUSNP: return "K3s-QEMU-SNP" + case K3sQEMUSNPGPU: + return "K3s-QEMU-SNP-GPU" case RKE2QEMUTDX: return "RKE2-QEMU-TDX" case MetalQEMUSNP: @@ -73,6 +77,8 @@ func FromString(s string) (Platform, error) { return K3sQEMUTDX, nil case "k3s-qemu-snp": return K3sQEMUSNP, nil + case "k3s-qemu-snp-gpu": + return K3sQEMUSNPGPU, nil case "rke2-qemu-tdx": return RKE2QEMUTDX, nil case "metal-qemu-snp": diff --git a/justfile b/justfile index ea650a47c..5f070c61f 100644 --- a/justfile +++ b/justfile @@ -47,7 +47,7 @@ node-installer platform=default_platform: just push "tardev-snapshotter" just push "node-installer-microsoft" ;; - "Metal-QEMU-SNP"|"Metal-QEMU-TDX"|"K3s-QEMU-SNP"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX") + "Metal-QEMU-SNP"|"Metal-QEMU-TDX"|"K3s-QEMU-SNP"|"K3s-QEMU-SNP-GPU"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX") just push "nydus-snapshotter" just push "node-installer-kata" ;; @@ -186,7 +186,7 @@ create-pre platform=default_platform: # TODO(burgerdev): this should create the resource group for consistency : ;; - "Metal-QEMU-SNP"|"Metal-QEMU-TDX"|"K3s-QEMU-SNP"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX") + "Metal-QEMU-SNP"|"Metal-QEMU-TDX"|"K3s-QEMU-SNP"|"K3s-QEMU-SNP-GPU"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX") : ;; "AKS-PEER-SNP") @@ -215,7 +215,7 @@ create platform=default_platform: "AKS-CLH-SNP") nix run -L .#scripts.create-coco-aks -- --name="$azure_resource_group" --location="$azure_location" ;; - "Metal-QEMU-SNP"|"Metal-QEMU-TDX"|"K3s-QEMU-SNP"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX") + "Metal-QEMU-SNP"|"Metal-QEMU-TDX"|"K3s-QEMU-SNP"|"K3s-QEMU-SNP-GPU"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX") : ;; "AKS-PEER-SNP") @@ -328,7 +328,7 @@ get-credentials platform=default_platform: "K3s-QEMU-TDX") nix run -L .#scripts.get-credentials "projects/796962942582/secrets/m50-ganondorf-kubeconf/versions/5" ;; - "K3s-QEMU-SNP") + "K3s-QEMU-SNP"|"K3s-QEMU-SNP-GPU") nix run -L .#scripts.get-credentials "projects/796962942582/secrets/discovery-kubeconf/versions/2" ;; *) @@ -352,7 +352,7 @@ destroy platform=default_platform: "AKS-CLH-SNP") nix run -L .#scripts.destroy-coco-aks -- --name="$azure_resource_group" ;; - "K3s-QEMU-SNP"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX") + "K3s-QEMU-SNP"|"K3s-QEMU-SNP-GPU"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX") : ;; "AKS-PEER-SNP") @@ -377,7 +377,7 @@ destroy-post platform=default_platform: # TODO(burgerdev): this should destroy the resource group for consistency. : ;; - "K3s-QEMU-SNP"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX") + "K3s-QEMU-SNP"|"K3s-QEMU-SNP-GPU"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX") : ;; "AKS-PEER-SNP") diff --git a/nodeinstaller/internal/config/kata_runtime_test.go b/nodeinstaller/internal/config/kata_runtime_test.go index 25e81fee9..bfef090fb 100644 --- a/nodeinstaller/internal/config/kata_runtime_test.go +++ b/nodeinstaller/internal/config/kata_runtime_test.go @@ -28,7 +28,7 @@ func TestKataConfig(t *testing.T) { assert.Contains(string(configBytes), "[Runtime]") switch platform { - case platforms.K3sQEMUSNP, platforms.K3sQEMUTDX, platforms.MetalQEMUSNP, platforms.MetalQEMUTDX, platforms.RKE2QEMUTDX: + case platforms.K3sQEMUSNP, platforms.K3sQEMUSNPGPU, platforms.K3sQEMUTDX, platforms.MetalQEMUSNP, platforms.MetalQEMUTDX, platforms.RKE2QEMUTDX: assert.Contains(string(configBytes), "[Hypervisor.qemu]") case platforms.AKSCloudHypervisorSNP: assert.Contains(string(configBytes), "[Hypervisor.clh]") diff --git a/nodeinstaller/internal/constants/constants.go b/nodeinstaller/internal/constants/constants.go index 106bc7065..adab55fa9 100644 --- a/nodeinstaller/internal/constants/constants.go +++ b/nodeinstaller/internal/constants/constants.go @@ -75,7 +75,7 @@ func KataRuntimeConfig(baseDir string, platform platforms.Platform, qemuExtraKer if debug { config.Hypervisor["qemu"]["enable_debug"] = true } - case platforms.MetalQEMUSNP, platforms.K3sQEMUSNP: + case platforms.MetalQEMUSNP, platforms.K3sQEMUSNP, platforms.K3sQEMUSNPGPU: if err := toml.Unmarshal([]byte(kataBareMetalQEMUSNPBaseConfig), &config); err != nil { return nil, fmt.Errorf("failed to unmarshal kata runtime configuration: %w", err) } diff --git a/nodeinstaller/node-installer.go b/nodeinstaller/node-installer.go index e54f0cb98..d4d2c52e2 100644 --- a/nodeinstaller/node-installer.go +++ b/nodeinstaller/node-installer.go @@ -113,7 +113,7 @@ func run(ctx context.Context, fetcher assetFetcher, platform platforms.Platform, case platforms.MetalQEMUTDX: kataConfigPath = filepath.Join(kataConfigPath, "configuration-qemu-tdx.toml") containerdConfigPath = filepath.Join(hostMount, "etc", "containerd", "config.toml") - case platforms.K3sQEMUSNP: + case platforms.K3sQEMUSNP, platforms.K3sQEMUSNPGPU: kataConfigPath = filepath.Join(kataConfigPath, "configuration-qemu-snp.toml") containerdConfigPath = filepath.Join(hostMount, "var", "lib", "rancher", "k3s", "agent", "etc", "containerd", "config.toml.tmpl") case platforms.K3sQEMUTDX: @@ -147,7 +147,7 @@ func run(ctx context.Context, fetcher assetFetcher, platform platforms.Platform, switch platform { case platforms.AKSCloudHypervisorSNP, platforms.MetalQEMUSNP, platforms.MetalQEMUTDX: return restartHostContainerd(containerdConfigPath, "containerd") - case platforms.K3sQEMUTDX, platforms.K3sQEMUSNP: + case platforms.K3sQEMUTDX, platforms.K3sQEMUSNP, platforms.K3sQEMUSNPGPU: if hostServiceExists("k3s") { return restartHostContainerd(containerdConfigPath, "k3s") } else if hostServiceExists("k3s-agent") { @@ -212,7 +212,7 @@ func patchContainerdConfig(runtimeHandler, basePath, configPath string, platform case platforms.AKSCloudHypervisorSNP: snapshotterName = fmt.Sprintf("tardev-%s", runtimeHandler) socketName = fmt.Sprintf("/run/containerd/tardev-snapshotter-%s.sock", runtimeHandler) - case platforms.MetalQEMUTDX, platforms.MetalQEMUSNP, platforms.K3sQEMUTDX, platforms.K3sQEMUSNP, platforms.RKE2QEMUTDX: + case platforms.MetalQEMUTDX, platforms.MetalQEMUSNP, platforms.K3sQEMUTDX, platforms.K3sQEMUSNP, platforms.K3sQEMUSNPGPU, platforms.RKE2QEMUTDX: snapshotterName = fmt.Sprintf("nydus-%s", runtimeHandler) socketName = fmt.Sprintf("/run/containerd/containerd-nydus-grpc-%s.sock", runtimeHandler) diff --git a/nodeinstaller/node-installer_test.go b/nodeinstaller/node-installer_test.go index 79f80ec34..a1d14a4fc 100644 --- a/nodeinstaller/node-installer_test.go +++ b/nodeinstaller/node-installer_test.go @@ -42,6 +42,10 @@ func TestPatchContainerdConfig(t *testing.T) { platform: platforms.K3sQEMUSNP, expected: expectedConfBareMetalQEMUSNP, }, + "BareMetalQEMUSNPGPU": { + platform: platforms.K3sQEMUSNPGPU, + expected: expectedConfBareMetalQEMUSNP, + }, "Unknown": { platform: platforms.Unknown, wantErr: true, diff --git a/packages/by-name/kata/kata-runtime/package.nix b/packages/by-name/kata/kata-runtime/package.nix index a2b7fa9af..cb92bf98f 100644 --- a/packages/by-name/kata/kata-runtime/package.nix +++ b/packages/by-name/kata/kata-runtime/package.nix @@ -177,8 +177,6 @@ buildGoModule rec { # For example, this command should do the job: # `journalctl -t kata -l --no-pager | grep launching | tail -1` passthru = { - inherit src; - cmdline = { default = "tsc=reliable no_timer_check rcupdate.rcu_expedited=1 i8042.direct=1 i8042.dumbkbd=1 i8042.nopnp=1 i8042.noaux=1 noreplace-smp reboot=k cryptomgr.notests net.ifnames=0 pci=lastbus=0 root=/dev/vda1 rootflags=ro rootfstype=erofs console=hvc0 console=hvc1 quiet systemd.show_status=false panic=1 nr_cpus=1 selinux=0 systemd.unit=kata-containers.target systemd.mask=systemd-networkd.service systemd.mask=systemd-networkd.socket scsi_mod.scan=none"; debug = "tsc=reliable no_timer_check rcupdate.rcu_expedited=1 i8042.direct=1 i8042.dumbkbd=1 i8042.nopnp=1 i8042.noaux=1 noreplace-smp reboot=k cryptomgr.notests net.ifnames=0 pci=lastbus=0 root=/dev/vda1 rootflags=ro rootfstype=erofs console=hvc0 console=hvc1 debug systemd.show_status=true systemd.log_level=debug panic=1 nr_cpus=1 selinux=0 systemd.unit=kata-containers.target systemd.mask=systemd-networkd.service systemd.mask=systemd-networkd.socket scsi_mod.scan=none agent.log=debug agent.debug_console agent.debug_console_vport=1026"; diff --git a/packages/scripts.nix b/packages/scripts.nix index 83ef304ba..35b5a0eda 100644 --- a/packages/scripts.nix +++ b/packages/scripts.nix @@ -259,7 +259,7 @@ cp ${pkgs.microsoft.genpolicy.settings-coordinator}/genpolicy-settings.json . ${pkgs.microsoft.genpolicy}/bin/genpolicy < "$tmpdir/coordinator_base.yml" ;; - "k3s-qemu-snp"|"k3s-qemu-tdx"|"rke2-qemu-tdx") + "k3s-qemu-snp"|"k3s-qemu-snp-gpu"|"k3s-qemu-tdx"|"rke2-qemu-tdx") cp ${pkgs.kata.genpolicy.rules-coordinator}/genpolicy-rules.rego rules.rego cp ${pkgs.kata.genpolicy.settings-coordinator}/genpolicy-settings.json . ${pkgs.kata.genpolicy}/bin/genpolicy < "$tmpdir/coordinator_base.yml"