use custom runtime "contrast-cc"

.github/workflows/e2e_openssl.yml

@@ -49,7 +49,7 @@ jobs:
           just get-credentials
       - name: Build and prepare deployments
         run: |
-          just coordinator initializer openssl port-forwarder
+          just coordinator initializer openssl port-forwarder node-installer
       - name: E2E Test
         run: |
           nix shell .#contrast.e2e --command openssl.test -test.v workspace/just.containerlookup

.github/workflows/e2e_servicemesh.yml

@@ -49,7 +49,7 @@ jobs:
           just get-credentials
       - name: Build and prepare deployments
         run: |
-          just coordinator initializer port-forwarder service-mesh-proxy
+          just coordinator initializer port-forwarder service-mesh-proxy node-installer
       - name: E2E Test
         run: |
           nix shell .#contrast.e2e --command servicemesh.test -test.v workspace/just.containerlookup

.github/workflows/release.yml

@@ -187,17 +187,27 @@ jobs:
       - name: Push containers with release tag
         run: |
           coordinatorImg=$(nix run .#containers.push-coordinator -- "$container_registry/contrast/coordinator")
+          nodeInstallerImg=$(nix run .#containers.push-node-installer -- "$container_registry/contrast/node-installer")
           nix run .#containers.push-initializer -- "$container_registry/contrast/initializer"
           echo "coordinatorImg=$coordinatorImg" | tee -a "$GITHUB_ENV"
+          echo "nodeInstallerImg=$nodeInstallerImg" | tee -a "$GITHUB_ENV"
       - name: Add tag to Coordinator image
         run: |
-          front=${coordinatorImg%@*}
-          back=${coordinatorImg#*@}
-          echo "coordinatorImgTagged=${front}:${{ inputs.version }}@${back}" | tee -a "$GITHUB_ENV"
+          frontCoord=${coordinatorImg%@*}
+          backCoord=${coordinatorImg#*@}
+          echo "coordinatorImgTagged=${frontCoord}:${{ inputs.version }}@${backCoord}" | tee -a "$GITHUB_ENV"
+          frontNodeInstaller=${nodeInstallerImg%@*}
+          backNodeInstaller=${nodeInstallerImg#*@}
+          echo "nodeInstallerImgTagged=${frontNodeInstaller}:${{ inputs.version }}@${backNodeInstaller}" | tee -a "$GITHUB_ENV"
       - name: Create portable coordinator resource definitions
         run: |
           mkdir -p workspace
           nix run .#scripts.write-coordinator-yaml -- "${coordinatorImgTagged}" > workspace/coordinator.yml
+          nix shell .#contrast --command resourcegen runtime workspace/runtime.yml
+          nix run .#kypatch images -- workspace/runtime.yml \
+            --replace "ghcr.io/edgelesssys/contrast/node-installer:latest" "$nodeInstallerImgTagged"
+          nix run .#kypatch namespace -- workspace/runtime.yml \
+            --replace edg-default kube-system
       - name: Update coordinator policy hash
         run: |
           yq < workspace/coordinator.yml \
@@ -221,6 +231,7 @@ jobs:
           files: |
             result-cli/bin/contrast
             workspace/coordinator.yml
+            workspace/runtime.yml
       - name: Reset temporary changes
         run: |
           git reset --hard ${{ needs.process-inputs.outputs.WORKING_BRANCH }}

cli/cmd/common.go

@@ -25,6 +25,10 @@ var (
 	defaultGenpolicySettings []byte
 	//go:embed assets/genpolicy-rules.rego
 	defaultRules []byte
+	// DefaultCoordinatorPolicyHash is derived from the coordinator release candidate and injected at release build time.
+	//
+	// It is intentionally left empty for dev builds.
+	DefaultCoordinatorPolicyHash = ""
 )
 
 func cachedir(subdir string) (string, error) {

cli/cmd/generate.go

@@ -166,24 +166,24 @@ func findGenerateTargets(args []string, logger *slog.Logger) ([]string, error) {
 		}
 	}
 
-	paths = filterNonCoCoRuntime("kata-cc-isolation", paths, logger)
+	paths = filterNonCoCoRuntime("contrast-cc", paths, logger)
 
 	if len(paths) == 0 {
 		return nil, fmt.Errorf("no .yml/.yaml files found")
 	}
 	return paths, nil
 }
 
-func filterNonCoCoRuntime(runtimeClassName string, paths []string, logger *slog.Logger) []string {
+func filterNonCoCoRuntime(runtimeClassNamePrefix string, paths []string, logger *slog.Logger) []string {
 	var filtered []string
 	for _, path := range paths {
 		data, err := os.ReadFile(path)
 		if err != nil {
 			logger.Warn("Failed to read file", "path", path, "err", err)
 			continue
 		}
-		if !bytes.Contains(data, []byte(runtimeClassName)) {
-			logger.Info("Ignoring non-CoCo runtime", "className", runtimeClassName, "path", path)
+		if !bytes.Contains(data, []byte(runtimeClassNamePrefix)) {
+			logger.Info("Ignoring non-CoCo runtime", "className", runtimeClassNamePrefix, "path", path)
 			continue
 		}
 		filtered = append(filtered, path)

cli/cmd/runtime.go

@@ -1,6 +1,29 @@
 package cmd
 
-// DefaultCoordinatorPolicyHash is derived from the coordinator release candidate and injected at release build time.
-//
-// It is intentionally left empty for dev builds.
-var DefaultCoordinatorPolicyHash = ""
+import (
+	"github.com/spf13/cobra"
+)
+
+// This value is injected at build time.
+var runtimeHandler = "contrast-cc"
+
+// NewRuntimeCmd creates the contrast runtime subcommand.
+func NewRuntimeCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "runtime",
+		Short: "Prints the runtimeClassName",
+		Long: `Prints runtimeClassName used by Contrast.
+
+Contrast uses a custom container runtime, where every pod is a confidential
+virtual machine. Pod specs of workloads running on Contrast must
+have the runtimeClassName set to the value returned by this command.
+		`,
+		Run: runRuntime,
+	}
+
+	return cmd
+}
+
+func runRuntime(cmd *cobra.Command, _ []string) {
+	cmd.Println(runtimeHandler)
+}

cli/main.go

@@ -41,6 +41,7 @@ func newRootCmd() *cobra.Command {
 		cmd.NewGenerateCmd(),
 		cmd.NewSetCmd(),
 		cmd.NewVerifyCmd(),
+		cmd.NewRuntimeCmd(),
 	)
 
 	return root

dev-docs/coco/policy.md

@@ -26,7 +26,7 @@ To trust the agent, we need to ensure that the agent only serves permitted reque
 For Contrast, the chain of trust looks like this:
 
 1. The CLI generates a policy and attaches it to the pod definition.
-2. Kubernetes schedules the pod on a node with `kata-cc-isolation` runtime.
+2. Kubernetes schedules the pod on a node with a CoCo runtime.
 3. Containerd takes the node, starts the Kata Shim and creates the pod sandbox.
 4. The Kata runtime starts a CVM with the policy's digest as `HOSTDATA`.
 5. The Kata runtime sets the policy using the `SetPolicy` method.

docs/docs/deployment.md

@@ -47,13 +47,14 @@ cp -R $MY_RESOURCE_DIR resources/
 </Tabs>
 
 To specify that a workload (pod, deployment, etc.) should be deployed as confidential containers,
-add `runtimeClassName: kata-cc-isolation` to the pod spec (pod definition or template).
+add `runtimeClassName: contrast-cc` to the pod spec (pod definition or template).
+This is a placeholder name that will be replaced by a versioned `runtimeClassName` when generating policies.
 In addition, add the Contrast Initializer as `initContainers` to these workloads and configure the
 workload to use the certificates written to a `volumeMount` named `tls-certs`.
 
 ```yaml
 spec: # v1.PodSpec
-  runtimeClassName: kata-cc-isolation
+  runtimeClassName: contrast-cc
   initContainers:
   - name: initializer
     image: "ghcr.io/edgelesssys/contrast/initializer:latest"

docs/docs/examples/emojivoto.md

@@ -57,7 +57,7 @@ contrast generate deployment/
 :::note[Runtime class and Initializer]
 
 The deployment YAML shipped for this demo is already configured to be used with Contrast.
-A runtime class `kata-cc-isolation` was added to the pods to signal they should be run
+A runtime class `contrast-cc-<VERSIONHASH>` was added to the pods to signal they should be run
 as Confidential Containers. In addition, the Contrast Initializer was added
 as an init container to these workloads to facilitate the attestation and certificate pulling
 before the actual workload is started.

e2e/internal/contrasttest/contrasttest.go

@@ -25,9 +25,10 @@ import (
 // ContrastTest is the Contrast test helper struct.
 type ContrastTest struct {
 	// inputs, usually filled by New()
-	Namespace  string
-	WorkDir    string
-	Kubeclient *kubeclient.Kubeclient
+	Namespace         string
+	WorkDir           string
+	ImageReplacements map[string]string
+	Kubeclient        *kubeclient.Kubeclient
 
 	// outputs of contrast subcommands
 	coordinatorPolicyHash string
@@ -36,16 +37,17 @@ type ContrastTest struct {
 }
 
 // New creates a new contrasttest.T object bound to the given test.
-func New(t *testing.T) *ContrastTest {
+func New(t *testing.T, imageReplacements map[string]string) *ContrastTest {
 	return &ContrastTest{
-		Namespace:  makeNamespace(t),
-		WorkDir:    t.TempDir(),
-		Kubeclient: kubeclient.NewForTest(t),
+		Namespace:         makeNamespace(t),
+		WorkDir:           t.TempDir(),
+		ImageReplacements: imageReplacements,
+		Kubeclient:        kubeclient.NewForTest(t),
 	}
 }
 
 // Init patches the given resources for the test environment and makes them available to Generate and Set.
-func (ct *ContrastTest) Init(t *testing.T, objs []*unstructured.Unstructured) {
+func (ct *ContrastTest) Init(t *testing.T, resources []any) {
 	require := require.New(t)
 
 	// Create namespace
@@ -65,17 +67,26 @@ func (ct *ContrastTest) Init(t *testing.T, objs []*unstructured.Unstructured) {
 		}
 	})
 
-	// Add the namespace for this test.
-	for _, obj := range objs {
-		require.NoError(ct.Kubeclient.PatchNamespace(ct.Namespace, obj))
+	// Prepare resources
+	resources = kuberesource.PatchImages(resources, ct.ImageReplacements)
+	resources = kuberesource.PatchNamespaces(resources, ct.Namespace)
+	unstructuredResources, err := kuberesource.ResourcesToUnstructured(resources)
+	require.NoError(err)
+	var objects []*unstructured.Unstructured
+	for _, obj := range unstructuredResources {
+		// TODO(burgerdev): remove once demo deployments don't contain namespaces anymore.
+		if obj.GetKind() == "Namespace" {
+			continue
+		}
+		objects = append(objects, obj)
 	}
 
-	// TODO(burgerdev): patch images
-
 	// Write resources to this test's tempdir.
-	buf, err := kuberesource.EncodeUnstructured(objs)
+	buf, err := kuberesource.EncodeUnstructured(objects)
 	require.NoError(err)
 	require.NoError(os.WriteFile(path.Join(ct.WorkDir, "resources.yaml"), buf, 0o644))
+
+	ct.installRuntime(t)
 }
 
 // Generate runs the contrast generate command.
@@ -192,6 +203,27 @@ func (ct *ContrastTest) commonArgs() []string {
 	}
 }
 
+// installRuntime initializes the kubernetes runtime class for the test.
+func (ct *ContrastTest) installRuntime(t *testing.T) {
+	require := require.New(t)
+
+	resources, err := kuberesource.Runtime()
+	require.NoError(err)
+
+	resources = kuberesource.PatchImages(resources, ct.ImageReplacements)
+	resources = kuberesource.PatchNamespaces(resources, ct.Namespace)
+
+	unstructuredResources, err := kuberesource.ResourcesToUnstructured(resources)
+	require.NoError(err)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+	defer cancel()
+
+	require.NoError(ct.Kubeclient.Apply(ctx, unstructuredResources...))
+
+	require.NoError(ct.Kubeclient.WaitForDaemonset(ctx, ct.Namespace, "contrast-node-installer"))
+}
+
 func makeNamespace(t *testing.T) string {
 	buf := make([]byte, 4)
 	re := regexp.MustCompile("[a-z0-9-]+")

e2e/internal/kubeclient/deploy.go

@@ -99,6 +99,53 @@ func (c *Kubeclient) WaitForDeployment(ctx context.Context, namespace, name stri
 	}
 }
 
+// WaitForDaemonset watches the given daemonset and blocks until the desired number of pods are
+// ready or the context expires (is cancelled or times out).
+func (c *Kubeclient) WaitForDaemonset(ctx context.Context, namespace, name string) error {
+	watcher, err := c.client.AppsV1().DaemonSets(namespace).Watch(ctx, metav1.ListOptions{FieldSelector: "metadata.name=" + name})
+	if err != nil {
+		return err
+	}
+	for {
+		select {
+		case evt := <-watcher.ResultChan():
+			switch evt.Type {
+			case watch.Added:
+				fallthrough
+			case watch.Modified:
+				ds, ok := evt.Object.(*appsv1.DaemonSet)
+				if !ok {
+					return fmt.Errorf("watcher received unexpected type %T", evt.Object)
+				}
+				if ds.Status.NumberReady >= ds.Status.DesiredNumberScheduled {
+					return nil
+				}
+			default:
+				return fmt.Errorf("unexpected watch event while waiting for daemonset %s/%s: %#v", namespace, name, evt.Object)
+			}
+		case <-ctx.Done():
+			logger := c.log.With("namespace", namespace)
+			logger.Error("daemonset did not become ready", "name", name, "contextErr", ctx.Err())
+			if ctx.Err() != context.DeadlineExceeded {
+				return ctx.Err()
+			}
+			// Fetch and print debug information.
+			ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+			defer cancel()
+			pods, err := c.PodsFromDaemonSet(ctx, namespace, name) //nolint:contextcheck // The parent context expired.
+			if err != nil {
+				logger.Error("could not fetch pods for daemonset", "name", name, "error", err)
+				return ctx.Err()
+			}
+			for _, pod := range pods {
+				if !isPodReady(&pod) {
+					logger.Debug("pod not ready", "name", pod.Name, "status", c.toJSON(pod.Status))
+				}
+			}
+		}
+	}
+}
+
 func (c *Kubeclient) toJSON(a any) string {
 	s, err := json.Marshal(a)
 	if err != nil {
@@ -167,22 +214,3 @@ func (c *Kubeclient) Delete(ctx context.Context, objects ...*unstructured.Unstru
 	}
 	return nil
 }
-
-// PatchNamespace adjusts the namespace of the given object in-place if it is an instance of a namespaced resource.
-func (c *Kubeclient) PatchNamespace(namespace string, obj *unstructured.Unstructured) error {
-	gvk := obj.GroupVersionKind()
-	resources, err := c.client.DiscoveryClient.ServerResourcesForGroupVersion(gvk.GroupVersion().String())
-	if err != nil {
-		return fmt.Errorf("API resources not found for %#v: %w", gvk, err)
-	}
-	for _, resource := range resources.APIResources {
-		if resource.Kind != obj.GetKind() {
-			continue
-		}
-		if resource.Namespaced {
-			obj.SetNamespace(namespace)
-		}
-		return nil
-	}
-	return fmt.Errorf("API resource not found for %#v", gvk)
-}

e2e/internal/kubeclient/kubeclient.go

@@ -114,6 +114,27 @@ func (c *Kubeclient) PodsFromDeployment(ctx context.Context, namespace, deployme
 	return out, nil
 }
 
+// PodsFromDaemonSet returns the pods from a daemonset in a namespace.
+//
+// A pod is considered to belong to a daemonset if it is owned by the DaemonSet in question.
+func (c *Kubeclient) PodsFromDaemonSet(ctx context.Context, namespace, daemonset string) ([]v1.Pod, error) {
+	pods, err := c.client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("listing pods: %w", err)
+	}
+
+	var out []v1.Pod
+	for _, pod := range pods.Items {
+		for _, ref := range pod.OwnerReferences {
+			if ref.Kind == "DaemonSet" && ref.Name == daemonset {
+				out = append(out, pod)
+			}
+		}
+	}
+
+	return out, nil
+}
+
 // Exec executes a process in a pod and returns the stdout and stderr.
 func (c *Kubeclient) Exec(ctx context.Context, namespace, pod string, argv []string) (
 	stdout string, stderr string, err error,

e2e/internal/kuberesource/parts.go

@@ -141,7 +141,7 @@ func Coordinator(namespace string) *CoordinatorConfig {
 				WithLabels(map[string]string{"app.kubernetes.io/name": "coordinator"}).
 				WithAnnotations(map[string]string{"contrast.edgeless.systems/pod-role": "coordinator"}).
 				WithSpec(PodSpec().
-					WithRuntimeClassName("kata-cc-isolation").
+					WithRuntimeClassName(runtimeHandler).
 					WithContainers(
 						Container().
 							WithName("coordinator").