diff --git a/node-installer/internal/constants/configuration-qemu-tdx.toml b/node-installer/internal/constants/configuration-qemu-tdx.toml new file mode 100644 index 0000000000..c37fd9b4ed --- /dev/null +++ b/node-installer/internal/constants/configuration-qemu-tdx.toml @@ -0,0 +1,61 @@ +# Minimized list, inactive options removed. +# upstream source: https://github.com/kata-containers/kata-containers/blob/0f2a4d202e90b39b50074725b2cfe9c3088a4e20/src/runtime/config/configuration-qemu-tdx.toml.in +[hypervisor.qemu] +path = "/usr/bin/qemu-system-x86_64" +kernel = "/opt/kata/share/kata-containers/vmlinuz-confidential.container" +image = "/opt/kata/share/kata-containers/kata-containers-confidential.img" +machine_type = "q35" +tdx_quote_generation_service_socket_port = 4050 +rootfs_type="erofs" +confidential_guest = true +enable_annotations = ["enable_iommu", "virtio_fs_extra_args", "kernel_params", "default_vcpus", "default_memory"] +valid_hypervisor_paths = ["/usr/bin/qemu-system-x86_64"] +kernel_params = "" +firmware = "/usr/share/ovmf/OVMF.fd" +firmware_volume = "" +machine_accelerators="" +cpu_features="-vmx-rdseed-exit,pmu=off" +default_vcpus = 1 +default_maxvcpus = 0 +default_bridges = 1 +default_memory = 2048 +default_maxmemory = 0 +disable_block_device_use = false +shared_fs = "virtio-9p" +virtio_fs_daemon = "/opt/kata/libexec/virtiofsd" +valid_virtio_fs_daemon_paths = ["/opt/kata/libexec/virtiofsd"] +virtio_fs_cache_size = 0 +virtio_fs_queue_size = 1024 +virtio_fs_extra_args = ["--thread-pool-size=1", "--announce-submounts"] +virtio_fs_cache = "auto" +block_device_driver = "virtio-scsi" +block_device_aio = "io_uring" +enable_iothreads = false +enable_vhost_user_store = false +vhost_user_store_path = "/var/run/kata-containers/vhost-user" +valid_vhost_user_store_paths = ["/var/run/kata-containers/vhost-user"] +vhost_user_reconnect_timeout_sec = 0 +valid_file_mem_backends = [""] +pflashes = [] +enable_debug = false +valid_entropy_sources = ["/dev/urandom","/dev/random",""] +disable_selinux=false +disable_guest_selinux=true + +[agent.kata] +enable_debug = false +kernel_modules=[] +debug_console_enabled = false +dial_timeout = 60 + +[runtime] +enable_debug = false +internetworking_model="tcfilter" +disable_guest_seccomp=true +sandbox_cgroup_only=false +static_sandbox_resource_mgmt=true +sandbox_bind_mounts=[] +vfio_mode="guest-kernel" +disable_guest_empty_dir=false +experimental=[] +create_container_timeout = 60 diff --git a/node-installer/internal/constants/constants.go b/node-installer/internal/constants/constants.go index d6ea5bd329..260057bce6 100644 --- a/node-installer/internal/constants/constants.go +++ b/node-installer/internal/constants/constants.go @@ -5,17 +5,26 @@ package constants import ( _ "embed" + "fmt" "path/filepath" "github.com/edgelesssys/contrast/node-installer/internal/config" + "github.com/edgelesssys/contrast/node-installer/platforms" "github.com/pelletier/go-toml/v2" ) var ( - // containerdRuntimeBaseConfig is the configuration file for the containerd runtime + // kataCLHSNPBaseConfig is the configuration file for the Kata runtime on AKS SEV-SNP + // with Cloud-Hypervisor. // //go:embed configuration-clh-snp.toml - containerdRuntimeBaseConfig string + kataCLHSNPBaseConfig string + + // kataBareMetalQEMUTDXBaseConfig is the configuration file for the Kata runtime on bare-metal TDX + // with QEMU. + // + //go:embed configuration-qemu-tdx.toml + kataBareMetalQEMUTDXBaseConfig string // containerdBaseConfig is the base configuration file for containerd // @@ -27,17 +36,40 @@ var ( const CRIFQDN = "io.containerd.grpc.v1.cri" // KataRuntimeConfig returns the Kata runtime configuration. -func KataRuntimeConfig(baseDir string, debug bool) config.KataRuntimeConfig { +func KataRuntimeConfig(baseDir string, platform platforms.Platform, debug bool) (*config.KataRuntimeConfig, error) { var config config.KataRuntimeConfig - if err := toml.Unmarshal([]byte(containerdRuntimeBaseConfig), &config); err != nil { - panic(err) // should never happen + switch platform { + case platforms.AKSCloudHypervisorSNP: + if err := toml.Unmarshal([]byte(kataCLHSNPBaseConfig), &config); err != nil { + return nil, fmt.Errorf("failed to unmarshal kata runtime configuration: %w", err) + } + config.Hypervisor["clh"]["path"] = filepath.Join(baseDir, "bin", "cloud-hypervisor-snp") + config.Hypervisor["clh"]["igvm"] = filepath.Join(baseDir, "share", "kata-containers-igvm.img") + config.Hypervisor["clh"]["image"] = filepath.Join(baseDir, "share", "kata-containers.img") + config.Hypervisor["clh"]["valid_hypervisor_paths"] = []string{filepath.Join(baseDir, "bin", "cloud-hypervisor-snp")} + config.Hypervisor["clh"]["enable_debug"] = debug + return &config, nil + case platforms.K3sQEMUTDX, platforms.RKE2QEMUTDX: + if err := toml.Unmarshal([]byte(kataBareMetalQEMUTDXBaseConfig), &config); err != nil { + return nil, fmt.Errorf("failed to unmarshal kata runtime configuration: %w", err) + } + config.Hypervisor["qemu"]["path"] = filepath.Join(baseDir, "bin", "qemu-system-x86_64") + config.Hypervisor["qemu"]["firmware"] = filepath.Join(baseDir, "shae", "OVMF_CODE.fd") + config.Hypervisor["qemu"]["firmware_volume"] = filepath.Join(baseDir, "share", "OVMF_VARS.fd") + config.Hypervisor["qemu"]["image"] = filepath.Join(baseDir, "share", "kata-containers.img") + config.Hypervisor["qemu"]["kernel"] = filepath.Join(baseDir, "share", "kata-kernel") + config.Hypervisor["qemu"]["valid_hypervisor_paths"] = []string{filepath.Join(baseDir, "bin", "qemu-system-x86_64")} + if debug { + config.Hypervisor["qemu"]["enable_debug"] = true + config.Hypervisor["qemu"]["kernel_params"] = " agent.log=debug initcall_debug" + config.Agent["kata"]["enable_debug"] = true + config.Agent["kata"]["debug_console_enabled"] = true + config.Runtime["enable_debug"] = true + } + return &config, nil + default: + return nil, fmt.Errorf("unsupported platform: %s", platform) } - config.Hypervisor["clh"]["path"] = filepath.Join(baseDir, "bin", "cloud-hypervisor-snp") - config.Hypervisor["clh"]["igvm"] = filepath.Join(baseDir, "share", "kata-containers-igvm.img") - config.Hypervisor["clh"]["image"] = filepath.Join(baseDir, "share", "kata-containers.img") - config.Hypervisor["clh"]["valid_hypervisor_paths"] = []string{filepath.Join(baseDir, "bin", "cloud-hypervisor-snp")} - config.Hypervisor["clh"]["enable_debug"] = debug - return config } // ContainerdBaseConfig returns the base containerd configuration. @@ -50,17 +82,29 @@ func ContainerdBaseConfig() config.ContainerdConfig { } // ContainerdRuntimeConfigFragment returns the containerd runtime configuration fragment. -func ContainerdRuntimeConfigFragment(baseDir string) config.Runtime { - return config.Runtime{ - Type: "io.containerd.contrast-cc.v2", - Path: filepath.Join(baseDir, "bin", "containerd-shim-contrast-cc-v2"), - PodAnnotations: []string{"io.katacontainers.*"}, - Options: map[string]any{ - "ConfigPath": filepath.Join(baseDir, "etc", "configuration-clh-snp.toml"), - }, +func ContainerdRuntimeConfigFragment(baseDir string, platform platforms.Platform) (*config.Runtime, error) { + cfg := config.Runtime{ + Type: "io.containerd.contrast-cc.v2", + Path: filepath.Join(baseDir, "bin", "containerd-shim-contrast-cc-v2"), + PodAnnotations: []string{"io.katacontainers.*"}, PrivilegedWithoutHostDevices: true, - Snapshotter: "tardev", } + + switch platform { + case platforms.AKSCloudHypervisorSNP: + cfg.Snapshotter = "tardev" + cfg.Options = map[string]any{ + "ConfigPath": filepath.Join(baseDir, "etc", "configuration-clh-snp.toml"), + } + case platforms.K3sQEMUTDX, platforms.RKE2QEMUTDX: + cfg.Options = map[string]any{ + "ConfigPath": filepath.Join(baseDir, "etc", "configuration-qemu-tdx.toml"), + } + default: + return nil, fmt.Errorf("unsupported platform: %s", platform) + } + + return &cfg, nil } // TardevSnapshotterConfigFragment returns the tardev snapshotter configuration fragment. diff --git a/node-installer/node-installer.go b/node-installer/node-installer.go index 34af81f364..c1926c8536 100644 --- a/node-installer/node-installer.go +++ b/node-installer/node-installer.go @@ -6,6 +6,7 @@ package main import ( "context" "encoding/json" + "flag" "fmt" "os" "os/exec" @@ -17,19 +18,40 @@ import ( "github.com/edgelesssys/contrast/node-installer/internal/asset" "github.com/edgelesssys/contrast/node-installer/internal/config" "github.com/edgelesssys/contrast/node-installer/internal/constants" + "github.com/edgelesssys/contrast/node-installer/platforms" "github.com/pelletier/go-toml/v2" ) func main() { + shouldRestartContainerd := flag.Bool("restart", true, "Restart containerd after the runtime installation to make the changes effective.") + flag.Parse() + + var platform platforms.Platform + if len(os.Args) < 2 { + // For now, fall back to the default platform (AKS Cloud Hypervisor SNP) if no platform is specified. + platform = platforms.AKSCloudHypervisorSNP + // TODO(msanft): Remove this fallback once the node-installer deployment is platform-aware. + // fmt.Println("Usage: node-installer ") + // os.Exit(1) + } else { + var err error + + platform, err = platforms.FromString(os.Args[1]) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + } + fetcher := asset.NewDefaultFetcher() - if err := run(context.Background(), fetcher); err != nil { + if err := run(context.Background(), fetcher, platform, *shouldRestartContainerd); err != nil { fmt.Println(err) os.Exit(1) } fmt.Println("Installation completed successfully.") } -func run(ctx context.Context, fetcher assetFetcher) error { +func run(ctx context.Context, fetcher assetFetcher, platform platforms.Platform, shouldRestartContainerd bool) error { configDir := envWithDefault("CONFIG_DIR", "/config") hostMount := envWithDefault("HOST_MOUNT", "/host") @@ -88,16 +110,73 @@ func run(ctx context.Context, fetcher assetFetcher) error { return fmt.Errorf("chmod %q: %w", item.Name(), err) } } - clhConfigPath := filepath.Join(hostMount, runtimeBase, "etc", "configuration-clh-snp.toml") - if err := containerdRuntimeConfig(runtimeBase, clhConfigPath, config.DebugRuntime); err != nil { - return fmt.Errorf("generating clh_config.toml: %w", err) + + kataConfigPath := filepath.Join(hostMount, runtimeBase, "etc") + var containerdConfigPath string + switch platform { + case platforms.AKSCloudHypervisorSNP: + kataConfigPath = filepath.Join(kataConfigPath, "configuration-clh-snp.toml") + containerdConfigPath = filepath.Join(hostMount, "etc", "containerd", "config.toml") + case platforms.K3sQEMUTDX: + kataConfigPath = filepath.Join(kataConfigPath, "configuration-qemu-tdx.toml") + containerdConfigPath = filepath.Join(hostMount, "var", "lib", "rancher", "k3s", "agent", "etc", "containerd", "config.toml") + case platforms.RKE2QEMUTDX: + kataConfigPath = filepath.Join(kataConfigPath, "configuration-qemu-tdx.toml") + containerdConfigPath = filepath.Join(hostMount, "var", "lib", "rancher", "rke2", "agent", "etc", "containerd", "config.toml") + default: + return fmt.Errorf("unsupported platform %q", platform) } - containerdConfigPath := filepath.Join(hostMount, "etc", "containerd", "config.toml") - if err := patchContainerdConfig(config.RuntimeHandlerName, runtimeBase, containerdConfigPath); err != nil { - return fmt.Errorf("patching containerd config: %w", err) + + if err := containerdRuntimeConfig(runtimeBase, kataConfigPath, platform, config.DebugRuntime); err != nil { + return fmt.Errorf("generating kata runtime configuration: %w", err) + } + + switch platform { + case platforms.AKSCloudHypervisorSNP: + // AKS or any external-containerd based K8s distro: We can just patch the existing containerd config at /etc/containerd/config.toml + if err := patchContainerdConfig(config.RuntimeHandlerName, runtimeBase, containerdConfigPath, platform); err != nil { + return fmt.Errorf("patching containerd configuration: %w", err) + } + case platforms.K3sQEMUTDX, platforms.RKE2QEMUTDX: + // K3s or RKE2: We need to extend the configuration template, which, in it's un-templated form, is non-TOML. + // Therefore just write the TOML configuration fragment ourselves and append it to the template file. + // This assumes that the user does not yet have a runtime with the same name configured himself, + // but as our runtimes are hash-named, this should be a safe assumption. + if err := patchContainerdConfigTemplate(config.RuntimeHandlerName, runtimeBase, containerdConfigPath, platform); err != nil { + return fmt.Errorf("patching containerd configuration: %w", err) + } + default: + return fmt.Errorf("unsupported platform %q", platform) } - return restartHostContainerd(containerdConfigPath) + // If the user opted to not have us restart containerd, we're done here. + if !shouldRestartContainerd { + return nil + } + + switch platform { + case platforms.AKSCloudHypervisorSNP: + return restartHostContainerd(containerdConfigPath, "containerd") + case platforms.K3sQEMUTDX: + if hostServiceExists("k3s") { + return restartHostContainerd(containerdConfigPath, "k3s") + } else if hostServiceExists("k3s-agent") { + return restartHostContainerd(containerdConfigPath, "k3s-agent") + } else { + return fmt.Errorf("neither k3s nor k3s-agent service found") + } + case platforms.RKE2QEMUTDX: + if hostServiceExists("rke2-server") { + return restartHostContainerd(containerdConfigPath, "rke2-server") + } else if hostServiceExists("rke2-agent") { + return restartHostContainerd(containerdConfigPath, "rke2-agent") + } else { + return fmt.Errorf("neither rke2-server nor rke2-agent service found") + } + + default: + return fmt.Errorf("unsupported platform %q", platform) + } } func envWithDefault(key, dflt string) string { @@ -108,36 +187,45 @@ func envWithDefault(key, dflt string) string { return value } -func containerdRuntimeConfig(basePath, configPath string, debugRuntime bool) error { - kataRuntimeConfig := constants.KataRuntimeConfig(basePath, debugRuntime) +func containerdRuntimeConfig(basePath, configPath string, platform platforms.Platform, debugRuntime bool) error { + kataRuntimeConfig, err := constants.KataRuntimeConfig(basePath, platform, debugRuntime) + if err != nil { + return fmt.Errorf("generating kata runtime config: %w", err) + } rawConfig, err := toml.Marshal(kataRuntimeConfig) if err != nil { - return err + return fmt.Errorf("marshaling kata runtime config: %w", err) } return os.WriteFile(configPath, rawConfig, os.ModePerm) } -func patchContainerdConfig(runtimeName, basePath, configPath string) error { +func patchContainerdConfig(runtimeName, basePath, configPath string, platform platforms.Platform) error { existingRaw, existing, err := parseExistingContainerdConfig(configPath) if err != nil { existing = constants.ContainerdBaseConfig() } - // Add tardev snapshotter - if existing.ProxyPlugins == nil { - existing.ProxyPlugins = make(map[string]config.ProxyPlugin) - } - if _, ok := existing.ProxyPlugins["tardev"]; !ok { - existing.ProxyPlugins["tardev"] = constants.TardevSnapshotterConfigFragment() + // Add tardev snapshotter, only required for AKS + if platform == platforms.AKSCloudHypervisorSNP { + if existing.ProxyPlugins == nil { + existing.ProxyPlugins = make(map[string]config.ProxyPlugin) + } + if _, ok := existing.ProxyPlugins["tardev"]; !ok { + existing.ProxyPlugins["tardev"] = constants.TardevSnapshotterConfigFragment() + } } // Add contrast-cc runtime runtimes := ensureMapPath(&existing.Plugins, constants.CRIFQDN, "containerd", "runtimes") - runtimes[runtimeName] = constants.ContainerdRuntimeConfigFragment(basePath) + containerdRuntimeConfig, err := constants.ContainerdRuntimeConfigFragment(basePath, platform) + if err != nil { + return fmt.Errorf("generating containerd runtime config: %w", err) + } + runtimes[runtimeName] = containerdRuntimeConfig rawConfig, err := toml.Marshal(existing) if err != nil { - return err + return fmt.Errorf("marshaling containerd config: %w", err) } if slices.Equal(existingRaw, rawConfig) { @@ -149,6 +237,36 @@ func patchContainerdConfig(runtimeName, basePath, configPath string) error { return os.WriteFile(configPath, rawConfig, os.ModePerm) } +func patchContainerdConfigTemplate(runtimeName, basePath, configTemplatePath string, platform platforms.Platform) error { + existingConfig, err := os.ReadFile(configTemplatePath) + if err != nil { + return fmt.Errorf("reading containerd config template: %w", err) + } + + // Extend a scratchpad config with the new plugin configuration. (including the new contrast-cc runtime) + var newConfigFragment config.ContainerdConfig + runtimes := ensureMapPath(&newConfigFragment.Plugins, constants.CRIFQDN, "containerd", "runtimes") + containerdRuntimeConfig, err := constants.ContainerdRuntimeConfigFragment(basePath, platform) + if err != nil { + return fmt.Errorf("generating containerd runtime config: %w", err) + } + runtimes[runtimeName] = containerdRuntimeConfig + + // We purposely don't marshal the full config, as we only want to append the plugin section. + rawNewPluginConfig, err := toml.Marshal(newConfigFragment.Plugins) + if err != nil { + return fmt.Errorf("marshaling containerd runtime config: %w", err) + } + + // First append the existing config template by a newline, so that if it ends without a newline, + // the new config fragment isn't appended to the last line.. + newRawConfig := append(existingConfig, []byte("\n")...) + // ..then append the new config fragment + newRawConfig = append(newRawConfig, rawNewPluginConfig...) + + return os.WriteFile(configTemplatePath, newRawConfig, os.ModePerm) +} + func parseExistingContainerdConfig(path string) ([]byte, config.ContainerdConfig, error) { configData, err := os.ReadFile(path) if err != nil { @@ -163,7 +281,7 @@ func parseExistingContainerdConfig(path string) ([]byte, config.ContainerdConfig return configData, cfg, nil } -func restartHostContainerd(containerdConfigPath string) error { +func restartHostContainerd(containerdConfigPath, service string) error { // get mtime of the config file info, err := os.Stat(containerdConfigPath) if err != nil { @@ -173,40 +291,48 @@ func restartHostContainerd(containerdConfigPath string) error { // get containerd start time // Note that "--timestamp=unix" is not supported in the installed version of systemd (v250) at the time of writing. - containerdStartTime, err := exec.Command( + serviceStartTime, err := exec.Command( "nsenter", "--target", "1", "--mount", "--", - "systemctl", "show", "--timestamp=utc", "--property=ActiveEnterTimestamp", "containerd", + "systemctl", "show", "--timestamp=utc", "--property=ActiveEnterTimestamp", service, ).CombinedOutput() if err != nil { - return fmt.Errorf("getting containerd start time: %w %q", err, containerdStartTime) + return fmt.Errorf("getting service (%s) start time: %w %q", service, err, serviceStartTime) } // format: ActiveEnterTimestamp=Day YYYY-MM-DD HH:MM:SS UTC - dayUTC := strings.TrimPrefix(strings.TrimSpace(string(containerdStartTime)), "ActiveEnterTimestamp=") + dayUTC := strings.TrimPrefix(strings.TrimSpace(string(serviceStartTime)), "ActiveEnterTimestamp=") startTime, err := time.Parse("Mon 2006-01-02 15:04:05 MST", dayUTC) if err != nil { - return fmt.Errorf("parsing containerd start time: %w", err) + return fmt.Errorf("parsing service (%s) start time: %w", service, err) } - fmt.Printf("containerd start time: %s\n", startTime.Format(time.RFC3339)) + fmt.Printf("service (%s) start time: %s\n", service, startTime.Format(time.RFC3339)) fmt.Printf("config mtime: %s\n", configMtime.Format(time.RFC3339)) if startTime.After(configMtime) { - fmt.Println("containerd already running with the newest config") + fmt.Printf("service (%s) already running with the newest config\n", service) return nil } // This command will restart containerd on the host and will take down the installer with it. out, err := exec.Command( "nsenter", "--target", "1", "--mount", "--", - "systemctl", "restart", "containerd", + "systemctl", "restart", service, ).CombinedOutput() if err != nil { - return fmt.Errorf("restarting containerd: %w: %s", err, out) + return fmt.Errorf("restarting service (%s): %w: %s", service, err, out) } - fmt.Printf("containerd restarted: %s\n", out) + fmt.Printf("service (%s) restarted: %s\n", service, out) return nil } +func hostServiceExists(service string) bool { + if err := exec.Command("nsenter", "--target", "1", "--mount", "--", + "systemctl", "status", service).Run(); err != nil { + return false + } + return true +} + // ensureMapPath ensures that the given path exists in the map and // returns the last map in the chain. func ensureMapPath(in *map[string]any, path ...string) map[string]any { diff --git a/node-installer/node-installer_test.go b/node-installer/node-installer_test.go index b51fadee9b..e28d0178dc 100644 --- a/node-installer/node-installer_test.go +++ b/node-installer/node-installer_test.go @@ -8,108 +8,63 @@ import ( "path/filepath" "testing" + _ "embed" + + "github.com/edgelesssys/contrast/node-installer/platforms" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestPatchContainerdConfig(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - tmpDir, err := os.MkdirTemp("", "patch-containerd-config-test") - require.NoError(err) - t.Cleanup(func() { _ = os.RemoveAll(tmpDir) }) - - configPath := filepath.Join(tmpDir, "config.toml") - - require.NoError(patchContainerdConfig("my-runtime", "/opt/edgeless/my-runtime", configPath)) - - configData, err := os.ReadFile(configPath) - require.NoError(err) - assert.Equal(`version = 2 -root = '' -state = '' -temp = '' -plugin_dir = '' -disabled_plugins = [] -required_plugins = [] -oom_score = 0 -imports = [] - -[metrics] -address = '0.0.0.0:10257' - -[plugins] -[plugins.'io.containerd.grpc.v1.cri'] -sandbox_image = 'mcr.microsoft.com/oss/kubernetes/pause:3.6' - -[plugins.'io.containerd.grpc.v1.cri'.cni] -bin_dir = '/opt/cni/bin' -conf_dir = '/etc/cni/net.d' -conf_template = '/etc/containerd/kubenet_template.conf' - -[plugins.'io.containerd.grpc.v1.cri'.containerd] -default_runtime_name = 'runc' -disable_snapshot_annotations = false - -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes] -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.kata] -runtime_type = 'io.containerd.kata.v2' - -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.kata-cc] -pod_annotations = ['io.katacontainers.*'] -privileged_without_host_devices = true -runtime_type = 'io.containerd.kata-cc.v2' -snapshotter = 'tardev' +var ( + //go:embed testdata/expected-aks-clh-snp.toml + expectedConfAKSCLHSNP []byte -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.kata-cc.options] -ConfigPath = '/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml' - -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.katacli] -runtime_type = 'io.containerd.runc.v1' - -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.katacli.options] -BinaryName = '/usr/bin/kata-runtime' -CriuPath = '' -IoGid = 0 -IoUid = 0 -NoNewKeyring = false -NoPivotRoot = false -Root = '' -ShimCgroup = '' -SystemdCgroup = false - -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.my-runtime] -runtime_type = 'io.containerd.contrast-cc.v2' -runtime_path = '/opt/edgeless/my-runtime/bin/containerd-shim-contrast-cc-v2' -pod_annotations = ['io.katacontainers.*'] -privileged_without_host_devices = true -snapshotter = 'tardev' - -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.my-runtime.options] -ConfigPath = '/opt/edgeless/my-runtime/etc/configuration-clh-snp.toml' - -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.runc] -runtime_type = 'io.containerd.runc.v2' - -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.runc.options] -BinaryName = '/usr/bin/runc' - -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.untrusted] -runtime_type = 'io.containerd.runc.v2' - -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.untrusted.options] -BinaryName = '/usr/bin/runc' - -[plugins.'io.containerd.grpc.v1.cri'.registry] -config_path = '/etc/containerd/certs.d' - -[plugins.'io.containerd.grpc.v1.cri'.registry.headers] -X-Meta-Source-Client = ['azure/aks'] + //go:embed testdata/expected-bare-metal-qemu-tdx.toml + expectedConfBareMetalQEMUTDX []byte +) -[proxy_plugins] -[proxy_plugins.tardev] -type = 'snapshot' -address = '/run/containerd/tardev-snapshotter.sock' -`, string(configData)) +func TestPatchContainerdConfig(t *testing.T) { + testCases := map[string]struct { + platform platforms.Platform + expected []byte + wantErr bool + }{ + "AKSCLHSNP": { + platform: platforms.AKSCloudHypervisorSNP, + expected: expectedConfAKSCLHSNP, + }, + "BareMetalQEMUTDX": { + platform: platforms.K3sQEMUTDX, + expected: expectedConfBareMetalQEMUTDX, + }, + "Unknown": { + platform: platforms.Unknown, + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + tmpDir, err := os.MkdirTemp("", "patch-containerd-config-test") + require.NoError(err) + t.Cleanup(func() { _ = os.RemoveAll(tmpDir) }) + + configPath := filepath.Join(tmpDir, "config.toml") + + err = patchContainerdConfig("my-runtime", "/opt/edgeless/my-runtime", + configPath, tc.platform) + if tc.wantErr { + require.Error(err) + return + } + require.NoError(err) + + configData, err := os.ReadFile(configPath) + require.NoError(err) + assert.Equal(tc.expected, configData) + }) + } } diff --git a/node-installer/testdata/expected-aks-clh-snp.toml b/node-installer/testdata/expected-aks-clh-snp.toml new file mode 100644 index 0000000000..3ffd1f423d --- /dev/null +++ b/node-installer/testdata/expected-aks-clh-snp.toml @@ -0,0 +1,85 @@ +version = 2 +root = '' +state = '' +temp = '' +plugin_dir = '' +disabled_plugins = [] +required_plugins = [] +oom_score = 0 +imports = [] + +[metrics] +address = '0.0.0.0:10257' + +[plugins] +[plugins.'io.containerd.grpc.v1.cri'] +sandbox_image = 'mcr.microsoft.com/oss/kubernetes/pause:3.6' + +[plugins.'io.containerd.grpc.v1.cri'.cni] +bin_dir = '/opt/cni/bin' +conf_dir = '/etc/cni/net.d' +conf_template = '/etc/containerd/kubenet_template.conf' + +[plugins.'io.containerd.grpc.v1.cri'.containerd] +default_runtime_name = 'runc' +disable_snapshot_annotations = false + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes] +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.kata] +runtime_type = 'io.containerd.kata.v2' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.kata-cc] +pod_annotations = ['io.katacontainers.*'] +privileged_without_host_devices = true +runtime_type = 'io.containerd.kata-cc.v2' +snapshotter = 'tardev' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.kata-cc.options] +ConfigPath = '/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.katacli] +runtime_type = 'io.containerd.runc.v1' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.katacli.options] +BinaryName = '/usr/bin/kata-runtime' +CriuPath = '' +IoGid = 0 +IoUid = 0 +NoNewKeyring = false +NoPivotRoot = false +Root = '' +ShimCgroup = '' +SystemdCgroup = false + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.my-runtime] +runtime_type = 'io.containerd.contrast-cc.v2' +runtime_path = '/opt/edgeless/my-runtime/bin/containerd-shim-contrast-cc-v2' +pod_annotations = ['io.katacontainers.*'] +privileged_without_host_devices = true +snapshotter = 'tardev' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.my-runtime.options] +ConfigPath = '/opt/edgeless/my-runtime/etc/configuration-clh-snp.toml' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.runc] +runtime_type = 'io.containerd.runc.v2' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.runc.options] +BinaryName = '/usr/bin/runc' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.untrusted] +runtime_type = 'io.containerd.runc.v2' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.untrusted.options] +BinaryName = '/usr/bin/runc' + +[plugins.'io.containerd.grpc.v1.cri'.registry] +config_path = '/etc/containerd/certs.d' + +[plugins.'io.containerd.grpc.v1.cri'.registry.headers] +X-Meta-Source-Client = ['azure/aks'] + +[proxy_plugins] +[proxy_plugins.tardev] +type = 'snapshot' +address = '/run/containerd/tardev-snapshotter.sock' diff --git a/node-installer/testdata/expected-bare-metal-qemu-tdx.toml b/node-installer/testdata/expected-bare-metal-qemu-tdx.toml new file mode 100644 index 0000000000..dc5d7b63ac --- /dev/null +++ b/node-installer/testdata/expected-bare-metal-qemu-tdx.toml @@ -0,0 +1,79 @@ +version = 2 +root = '' +state = '' +temp = '' +plugin_dir = '' +disabled_plugins = [] +required_plugins = [] +oom_score = 0 +imports = [] + +[metrics] +address = '0.0.0.0:10257' + +[plugins] +[plugins.'io.containerd.grpc.v1.cri'] +sandbox_image = 'mcr.microsoft.com/oss/kubernetes/pause:3.6' + +[plugins.'io.containerd.grpc.v1.cri'.cni] +bin_dir = '/opt/cni/bin' +conf_dir = '/etc/cni/net.d' +conf_template = '/etc/containerd/kubenet_template.conf' + +[plugins.'io.containerd.grpc.v1.cri'.containerd] +default_runtime_name = 'runc' +disable_snapshot_annotations = false + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes] +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.kata] +runtime_type = 'io.containerd.kata.v2' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.kata-cc] +pod_annotations = ['io.katacontainers.*'] +privileged_without_host_devices = true +runtime_type = 'io.containerd.kata-cc.v2' +snapshotter = 'tardev' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.kata-cc.options] +ConfigPath = '/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.katacli] +runtime_type = 'io.containerd.runc.v1' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.katacli.options] +BinaryName = '/usr/bin/kata-runtime' +CriuPath = '' +IoGid = 0 +IoUid = 0 +NoNewKeyring = false +NoPivotRoot = false +Root = '' +ShimCgroup = '' +SystemdCgroup = false + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.my-runtime] +runtime_type = 'io.containerd.contrast-cc.v2' +runtime_path = '/opt/edgeless/my-runtime/bin/containerd-shim-contrast-cc-v2' +pod_annotations = ['io.katacontainers.*'] +privileged_without_host_devices = true + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.my-runtime.options] +ConfigPath = '/opt/edgeless/my-runtime/etc/configuration-qemu-tdx.toml' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.runc] +runtime_type = 'io.containerd.runc.v2' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.runc.options] +BinaryName = '/usr/bin/runc' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.untrusted] +runtime_type = 'io.containerd.runc.v2' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.untrusted.options] +BinaryName = '/usr/bin/runc' + +[plugins.'io.containerd.grpc.v1.cri'.registry] +config_path = '/etc/containerd/certs.d' + +[plugins.'io.containerd.grpc.v1.cri'.registry.headers] +X-Meta-Source-Client = ['azure/aks']