cli: output coordinator policy on generate

As soon as we embed a default coordinator policy into the CLI, the set
and verify subcommands will stop working if the coordinator policy does
not match expectations. Thus, we need to make users aware if we detect a
non-default coordinator among the deployments.

If the CLI encounters an unexpected coordinator policy in the input
YAML, it prints the policy hash to stdout and warns on stderr. This
provides the user with two choices, depending on their workflow:

1. Users that don't expect changes to the coordinator policy can abort
   right away to check why the policy changed. But even if they ignore
   the output entirely, set will still fail closed.
2. Users that deliberately changed the coordinator but want to verify it
   nonetheless. These users need to pass the output to the set command,
   if it was not empty.

Co-authored-by: Paul Meyer <[email protected]>

README.md

@@ -94,6 +94,8 @@ spec:
     metadata:
       labels:
         app.kubernetes.io/name: coordinator
+      annotations:
+        nunki.edgeless.systems/pod-role: coordinator
     spec:
       runtimeClassName: kata-cc-isolation
       containers:

cli/generate.go

@@ -19,7 +19,10 @@ import (
 	"github.com/spf13/cobra"
 )
 
-const kataPolicyAnnotationKey = "io.katacontainers.config.agent.policy"
+const (
+	kataPolicyAnnotationKey = "io.katacontainers.config.agent.policy"
+	nunkiRoleAnnotationKey  = "nunki.edgeless.systems/pod-role"
+)
 
 func newGenerateCmd() *cobra.Command {
 	cmd := &cobra.Command{
@@ -31,9 +34,14 @@ func newGenerateCmd() *cobra.Command {
 		This will download the referenced container images to calculate the dm-verity
 		hashes of the image layers. In addition, the Rego policy will be used as base
 		and updated with the given settings file. For each container workload, the policy
-		is added as annotaiton in the Kubernetes YAML.
+		is added as an annotation to the Kubernetes YAML.
 
 		The hashes of the policies are added to the manifest.
+
+		If the Kubernetes YAML contains a Nunki Coordinator pod whose policy differs from
+		the embedded default, the generated policy will be printed to stdout, alongside a
+		warning message on stderr. This hash needs to be passed to the set and verify
+		subcommands.
 		`,
 		RunE: runGenerate,
 	}
@@ -98,6 +106,10 @@ func runGenerate(cmd *cobra.Command, args []string) error {
 
 	log.Info("Updated manifest", "path", flags.manifestPath)
 
+	if hash := getCoordinatorPolicyHash(policies, log); hash != "" {
+		fmt.Fprintln(cmd.OutOrStdout(), hash)
+	}
+
 	return nil
 }
 

cli/policies.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"log/slog"
 	"os"
 	"slices"
 
@@ -25,23 +26,28 @@ func policiesFromKubeResources(yamlPaths []string) (map[string]deployment, error
 
 	deployments := make(map[string]deployment)
 	for _, objAny := range kubeObjs {
-		var name, annotation string
+		var name, annotation, role string
 		switch obj := objAny.(type) {
 		case kubeapi.Pod:
 			name = obj.Name
 			annotation = obj.Annotations[kataPolicyAnnotationKey]
+			role = obj.Annotations[nunkiRoleAnnotationKey]
 		case kubeapi.Deployment:
 			name = obj.Name
 			annotation = obj.Spec.Template.Annotations[kataPolicyAnnotationKey]
+			role = obj.Spec.Template.Annotations[nunkiRoleAnnotationKey]
 		case kubeapi.ReplicaSet:
 			name = obj.Name
 			annotation = obj.Spec.Template.Annotations[kataPolicyAnnotationKey]
+			role = obj.Spec.Template.Annotations[nunkiRoleAnnotationKey]
 		case kubeapi.StatefulSet:
 			name = obj.Name
 			annotation = obj.Spec.Template.Annotations[kataPolicyAnnotationKey]
+			role = obj.Spec.Template.Annotations[nunkiRoleAnnotationKey]
 		case kubeapi.DaemonSet:
 			name = obj.Name
 			annotation = obj.Spec.Template.Annotations[kataPolicyAnnotationKey]
+			role = obj.Spec.Template.Annotations[nunkiRoleAnnotationKey]
 		}
 		if annotation == "" {
 			continue
@@ -56,6 +62,7 @@ func policiesFromKubeResources(yamlPaths []string) (map[string]deployment, error
 		deployments[name] = deployment{
 			name:   name,
 			policy: policy,
+			role:   role,
 		}
 	}
 
@@ -91,9 +98,27 @@ func checkPoliciesMatchManifest(policies map[string]deployment, policyHashes map
 	return nil
 }
 
+// getCoordinatorPolicyHash returns the policy hash for the Nunki coordinator among the given deployments.
+//
+// If the deployments contain a coordinator, that coordinator's policy hash is returned, otherwise
+// an empty string is returned.
+//
+// If there is more than one coordinator, it's unspecified which one will be used.
+func getCoordinatorPolicyHash(policies map[string]deployment, log *slog.Logger) string {
+	var hash string
+	for _, deployment := range policies {
+		if deployment.role == "coordinator" {
+			log.Warn("Found unexpected coordinator policy", "name", deployment.name, "hash", deployment.policy.Hash())
+			hash = deployment.policy.Hash().String()
+		}
+	}
+	return hash
+}
+
 type deployment struct {
 	name   string
 	policy manifest.Policy
+	role   string
 }
 
 func (d deployment) DNSNames() []string {

deployments/emojivoto/coordinator.yml

@@ -12,6 +12,8 @@ spec:
     metadata:
       labels:
         app.kubernetes.io/name: coordinator
+      annotations:
+        nunki.edgeless.systems/pod-role: coordinator
     spec:
       runtimeClassName: kata-cc-isolation
       containers:

deployments/openssl/coordinator.yml

@@ -12,6 +12,8 @@ spec:
     metadata:
       labels:
         app.kubernetes.io/name: coordinator
+      annotations:
+        nunki.edgeless.systems/pod-role: coordinator
     spec:
       runtimeClassName: kata-cc-isolation
       containers:

deployments/simple/coordinator.yml

@@ -12,6 +12,8 @@ spec:
     metadata:
       labels:
         app.kubernetes.io/name: coordinator
+      annotations:
+        nunki.edgeless.systems/pod-role: coordinator
     spec:
       runtimeClassName: kata-cc-isolation
       containers:

justfile

@@ -41,7 +41,7 @@ generate target=default_deploy_target:
         -m ./{{ workspace_dir }}/manifest.json \
         -p ./{{ workspace_dir }} \
         -s genpolicy-msft.json \
-        ./{{ workspace_dir }}/deployment/*.yml
+        ./{{ workspace_dir }}/deployment/*.yml > ./{{ workspace_dir }}/just.coordinator-policy-hash
     duration=$(( $(date +%s) - $t ))
     echo "Generated policies in $duration seconds."
     echo "generate $duration" >> ./{{ workspace_dir }}/just.perf
@@ -79,7 +79,7 @@ set:
     PID=$!
     trap "kill $PID" EXIT
     nix run .#wait-for-port-listen -- 1313
-    policy=$(nix run .#get-coordinator-policy-hash -- ./{{ workspace_dir }}/deployment/*.yml)
+    policy=$(<./{{ workspace_dir }}/just.coordinator-policy-hash)
     t=$(date +%s)
     nix run .#cli -- set \
         -m ./{{ workspace_dir }}/manifest.json \

packages/default.nix

@@ -263,16 +263,4 @@ rec {
       exit 1
     '';
   };
-
-  get-coordinator-policy-hash = writeShellApplication {
-    name = "get-coordinator-policy-hash";
-    runtimeInputs = [ yq-go ];
-    text = ''
-      set -o pipefail
-      yq -e eval-all \
-        'select(.kind == "Deployment" and .metadata.name == "coordinator") |
-         .spec.template.metadata.annotations["io.katacontainers.config.agent.policy"]' "$@" |
-        base64 -d | sha256sum | cut -d' ' -f1
-    '';
-  };
 }