From 75706e902acdccaa4529dbb4a5ffd268ba98fc51 Mon Sep 17 00:00:00 2001
From: Paul Meyer <49727155+katexochen@users.noreply.github.com>
Date: Tue, 9 Jul 2024 17:15:49 +0200
Subject: [PATCH] node-installer: configure and run tardev-snapshotter

Signed-off-by: Paul Meyer <49727155+katexochen@users.noreply.github.com>
---
 internal/kuberesource/parts.go                | 37 +++++++++++++++----
 justfile                                      |  2 +-
 .../internal/constants/constants.go           | 12 +-----
 node-installer/node-installer.go              | 12 ++++--
 node-installer/node-installer_test.go         |  2 +-
 .../testdata/expected-aks-clh-snp.toml        |  6 +--
 6 files changed, 45 insertions(+), 26 deletions(-)

diff --git a/internal/kuberesource/parts.go b/internal/kuberesource/parts.go
index 1c9118e01e..c060b7cb57 100644
--- a/internal/kuberesource/parts.go
+++ b/internal/kuberesource/parts.go
@@ -36,12 +36,12 @@ type NodeInstallerConfig struct {
 func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstallerConfig, error) {
 	name := "contrast-node-installer"
 
-	var imageURL string
+	var nodeInstallerImageURL string
 	switch platform {
 	case platforms.AKSCloudHypervisorSNP:
-		imageURL = "ghcr.io/edgelesssys/contrast/node-installer-microsoft:latest"
+		nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-microsoft:latest"
 	case platforms.K3sQEMUTDX, platforms.RKE2QEMUTDX:
-		imageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata:latest"
+		nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata:latest"
 	default:
 		return nil, fmt.Errorf("unsupported platform %q", platform)
 	}
@@ -62,7 +62,7 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle
 					WithHostPID(true).
 					WithInitContainers(Container().
 						WithName("installer").
-						WithImage(imageURL).
+						WithImage(nodeInstallerImageURL).
 						WithResources(ResourceRequirements().
 							WithMemoryLimitAndRequest(100),
 						).
@@ -74,10 +74,27 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle
 					).
 					WithContainers(
 						Container().
-							WithName("pause").
-							WithImage("k8s.gcr.io/pause").
+							WithName("tardev-snapshotter").
+							WithImage("ghcr.io/edgelesssys/contrast/tardev-snapshotter:latest").
 							WithResources(ResourceRequirements().
-								WithMemoryLimitAndRequest(10),
+								WithMemoryLimitAndRequest(800),
+							).
+							WithVolumeMounts(
+								VolumeMount().
+									WithName("host-mount").
+									WithMountPath("/host"),
+								VolumeMount().
+									WithName("var-lib-containerd").
+									WithMountPath("/var/lib/containerd"),
+							).
+							WithArgs(
+								"tardev-snapshotter",
+								fmt.Sprintf("/var/lib/containerd/io.containerd.snapshotter.v1.tardev-%s", runtimeHandler),
+								fmt.Sprintf("/host/run/containerd/tardev-snapshotter-%s.sock", runtimeHandler),
+								"/host/var/run/containerd/containerd.sock",
+							).
+							WithEnv(
+								NewEnvVar("RUST_LOG", "tardev_snapshotter=trace"),
 							),
 					).
 					WithVolumes(
@@ -87,6 +104,12 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle
 								WithPath("/").
 								WithType(corev1.HostPathDirectory),
 							),
+						Volume().
+							WithName("var-lib-containerd").
+							WithHostPath(HostPathVolumeSource().
+								WithPath("/var/lib/containerd").
+								WithType(corev1.HostPathDirectory),
+							),
 					),
 				),
 			),
diff --git a/justfile b/justfile
index eada02724b..b8526c7ad7 100644
--- a/justfile
+++ b/justfile
@@ -33,7 +33,7 @@ default_platform := "AKS-CLH-SNP"
 workspace_dir := "workspace"
 
 # Build the node-installer, containerize and push it.
-node-installer platform=default_platform:
+node-installer platform=default_platform: tardev-snapshotter
     #!/usr/bin/env bash
     case {{ platform }} in
         "AKS-CLH-SNP")
diff --git a/node-installer/internal/constants/constants.go b/node-installer/internal/constants/constants.go
index 085e7547d5..db97eca812 100644
--- a/node-installer/internal/constants/constants.go
+++ b/node-installer/internal/constants/constants.go
@@ -81,7 +81,7 @@ func ContainerdBaseConfig() config.ContainerdConfig {
 }
 
 // ContainerdRuntimeConfigFragment returns the containerd runtime configuration fragment.
-func ContainerdRuntimeConfigFragment(baseDir string, platform platforms.Platform) (*config.Runtime, error) {
+func ContainerdRuntimeConfigFragment(baseDir, snapshotter string, platform platforms.Platform) (*config.Runtime, error) {
 	cfg := config.Runtime{
 		Type:                         "io.containerd.contrast-cc.v2",
 		Path:                         filepath.Join(baseDir, "bin", "containerd-shim-contrast-cc-v2"),
@@ -91,7 +91,7 @@ func ContainerdRuntimeConfigFragment(baseDir string, platform platforms.Platform
 
 	switch platform {
 	case platforms.AKSCloudHypervisorSNP:
-		cfg.Snapshotter = "tardev"
+		cfg.Snapshotter = snapshotter
 		cfg.Options = map[string]any{
 			"ConfigPath": filepath.Join(baseDir, "etc", "configuration-clh-snp.toml"),
 		}
@@ -105,11 +105,3 @@ func ContainerdRuntimeConfigFragment(baseDir string, platform platforms.Platform
 
 	return &cfg, nil
 }
-
-// TardevSnapshotterConfigFragment returns the tardev snapshotter configuration fragment.
-func TardevSnapshotterConfigFragment() config.ProxyPlugin {
-	return config.ProxyPlugin{
-		Type:    "snapshot",
-		Address: "/run/containerd/tardev-snapshotter.sock",
-	}
-}
diff --git a/node-installer/node-installer.go b/node-installer/node-installer.go
index 6e40db93ae..3f666ecbe5 100644
--- a/node-installer/node-installer.go
+++ b/node-installer/node-installer.go
@@ -199,19 +199,23 @@ func patchContainerdConfig(runtimeName, basePath, configPath string, platform pl
 		existing = constants.ContainerdBaseConfig()
 	}
 
+	snapshotterName := "no-snapshotter"
 	// Add tardev snapshotter, only required for AKS
 	if platform == platforms.AKSCloudHypervisorSNP {
 		if existing.ProxyPlugins == nil {
 			existing.ProxyPlugins = make(map[string]config.ProxyPlugin)
 		}
-		if _, ok := existing.ProxyPlugins["tardev"]; !ok {
-			existing.ProxyPlugins["tardev"] = constants.TardevSnapshotterConfigFragment()
+		snapshotterName = fmt.Sprintf("tardev-%s", runtimeName)
+		socketName := fmt.Sprintf("/run/containerd/tardev-snapshotter-%s.sock", runtimeName)
+		existing.ProxyPlugins[snapshotterName] = config.ProxyPlugin{
+			Type:    "snapshot",
+			Address: socketName,
 		}
 	}
 
 	// Add contrast-cc runtime
 	runtimes := ensureMapPath(&existing.Plugins, constants.CRIFQDN, "containerd", "runtimes")
-	containerdRuntimeConfig, err := constants.ContainerdRuntimeConfigFragment(basePath, platform)
+	containerdRuntimeConfig, err := constants.ContainerdRuntimeConfigFragment(basePath, snapshotterName, platform)
 	if err != nil {
 		return fmt.Errorf("generating containerd runtime config: %w", err)
 	}
@@ -240,7 +244,7 @@ func patchContainerdConfigTemplate(runtimeName, basePath, configTemplatePath str
 	// Extend a scratchpad config with the new plugin configuration. (including the new contrast-cc runtime)
 	var newConfigFragment config.ContainerdConfig
 	runtimes := ensureMapPath(&newConfigFragment.Plugins, constants.CRIFQDN, "containerd", "runtimes")
-	containerdRuntimeConfig, err := constants.ContainerdRuntimeConfigFragment(basePath, platform)
+	containerdRuntimeConfig, err := constants.ContainerdRuntimeConfigFragment(basePath, "no-snapshotter", platform)
 	if err != nil {
 		return fmt.Errorf("generating containerd runtime config: %w", err)
 	}
diff --git a/node-installer/node-installer_test.go b/node-installer/node-installer_test.go
index e28d0178dc..16096fe15e 100644
--- a/node-installer/node-installer_test.go
+++ b/node-installer/node-installer_test.go
@@ -64,7 +64,7 @@ func TestPatchContainerdConfig(t *testing.T) {
 
 			configData, err := os.ReadFile(configPath)
 			require.NoError(err)
-			assert.Equal(tc.expected, configData)
+			assert.Equal(string(tc.expected), string(configData))
 		})
 	}
 }
diff --git a/node-installer/testdata/expected-aks-clh-snp.toml b/node-installer/testdata/expected-aks-clh-snp.toml
index 3ffd1f423d..d3413bddec 100644
--- a/node-installer/testdata/expected-aks-clh-snp.toml
+++ b/node-installer/testdata/expected-aks-clh-snp.toml
@@ -56,7 +56,7 @@ runtime_type = 'io.containerd.contrast-cc.v2'
 runtime_path = '/opt/edgeless/my-runtime/bin/containerd-shim-contrast-cc-v2'
 pod_annotations = ['io.katacontainers.*']
 privileged_without_host_devices = true
-snapshotter = 'tardev'
+snapshotter = 'tardev-my-runtime'
 
 [plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.my-runtime.options]
 ConfigPath = '/opt/edgeless/my-runtime/etc/configuration-clh-snp.toml'
@@ -80,6 +80,6 @@ config_path = '/etc/containerd/certs.d'
 X-Meta-Source-Client = ['azure/aks']
 
 [proxy_plugins]
-[proxy_plugins.tardev]
+[proxy_plugins.tardev-my-runtime]
 type = 'snapshot'
-address = '/run/containerd/tardev-snapshotter.sock'
+address = '/run/containerd/tardev-snapshotter-my-runtime.sock'