peer-pods: pass policy hash via userdata

packages/by-name/cloud-api-adaptor/0001-measure-agent-config.toml-into-PCR-10.patch

@@ -0,0 +1,126 @@
+From 210219ab9c0a992247857fa2c2b60b01f6e6856f Mon Sep 17 00:00:00 2001
+From: Markus Rudy <[email protected]>
+Date: Wed, 23 Oct 2024 10:50:46 +0200
+Subject: [PATCH] measure agent-config.toml into PCR 10
+
+The agent config is security critical and needs to be measurable. It
+would be nice to measure the daemon config, too, but it contains
+hard-to-predict networking configuration (such as the node IP).
+
+This commit removes the files not relevant for Contrast out of an
+abundance of caution.
+---
+ src/cloud-api-adaptor/go.mod                  |  5 +++-
+ src/cloud-api-adaptor/go.sum                  |  2 ++
+ .../pkg/userdata/provision.go                 | 30 ++++++++++++++++---
+ 3 files changed, 32 insertions(+), 5 deletions(-)
+
+diff --git a/src/cloud-api-adaptor/go.mod b/src/cloud-api-adaptor/go.mod
+index bd419f6..011870a 100644
+--- a/src/cloud-api-adaptor/go.mod
++++ b/src/cloud-api-adaptor/go.mod
+@@ -1,6 +1,8 @@
+ module github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor
+
+-go 1.21
++go 1.22
++
++toolchain go1.23.2
+
+ require (
+ 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
+@@ -52,6 +54,7 @@ require (
+ 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f
+ 	github.com/docker/docker v25.0.5+incompatible
+ 	github.com/golang-jwt/jwt/v5 v5.2.1
++	github.com/google/go-tpm v0.9.1
+ 	github.com/moby/sys/mountinfo v0.7.1
+ 	github.com/pelletier/go-toml/v2 v2.1.0
+ 	github.com/sirupsen/logrus v1.9.3
+diff --git a/src/cloud-api-adaptor/go.sum b/src/cloud-api-adaptor/go.sum
+index 0dd05c2..1ffa1f8 100644
+--- a/src/cloud-api-adaptor/go.sum
++++ b/src/cloud-api-adaptor/go.sum
+@@ -322,6 +322,8 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
+ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
++github.com/google/go-tpm v0.9.1 h1:0pGc4X//bAlmZzMKf8iz6IsDo1nYTbYJ6FZN/rg4zdM=
++github.com/google/go-tpm v0.9.1/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
+ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
+ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+diff --git a/src/cloud-api-adaptor/pkg/userdata/provision.go b/src/cloud-api-adaptor/pkg/userdata/provision.go
+index 5c3b6ca..e741699 100644
+--- a/src/cloud-api-adaptor/pkg/userdata/provision.go
++++ b/src/cloud-api-adaptor/pkg/userdata/provision.go
+@@ -2,6 +2,7 @@ package userdata
+
+ import (
+ 	"context"
++	"crypto/sha256"
+ 	"fmt"
+ 	"log"
+ 	"os"
+@@ -12,6 +13,8 @@ import (
+ 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/aws"
+ 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/azure"
+ 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/docker"
++	"github.com/google/go-tpm/legacy/tpm2"
++	"github.com/google/go-tpm/tpmutil"
+ 	"gopkg.in/yaml.v2"
+ )
+
+@@ -162,6 +165,7 @@ func findConfigEntry(path string, cc *CloudConfig) []byte {
+ type entry struct {
+ 	path     string
+ 	optional bool
++	pcrIndex *int
+ }
+
+ func (f *entry) writeFile(cc *CloudConfig) error {
+@@ -179,6 +183,10 @@ func (f *entry) writeFile(cc *CloudConfig) error {
+ 		return fmt.Errorf("failed to create directory: %w", err)
+ 	}
+
++	if f.pcrIndex != nil {
++		extendPCR(*f.pcrIndex, bytes)
++	}
++
+ 	err = os.WriteFile(f.path, bytes, 0644)
+ 	if err != nil {
+ 		return fmt.Errorf("failed to write file: %w", err)
+@@ -189,11 +197,8 @@ func (f *entry) writeFile(cc *CloudConfig) error {
+
+ func processCloudConfig(cfg *Config, cc *CloudConfig) error {
+ 	entries := []entry{
+-		{path: cfg.paths.agentConfig, optional: false},
++		{path: cfg.paths.agentConfig, optional: false, pcrIndex: toPtr(10)},
+ 		{path: cfg.paths.daemonConfig, optional: false},
+-		{path: cfg.paths.aaConfig, optional: true},
+-		{path: cfg.paths.cdhConfig, optional: true},
+-		{path: cfg.paths.authJson, optional: true},
+ 	}
+
+ 	for _, e := range entries {
+@@ -228,3 +233,20 @@ func ProvisionFiles(cfg *Config) error {
+
+ 	return nil
+ }
++
++func extendPCR(pcrIndex int, data []byte) error {
++	digest := sha256.Sum256(data)
++
++	handle, err := tpm2.OpenTPM()
++	if err != nil {
++		return fmt.Errorf("opening TPM device: %w", err)
++	}
++	if err := tpm2.PCRExtend(handle, tpmutil.Handle(pcrIndex), tpm2.AlgSHA256, digest[:], ""); err != nil {
++		return fmt.Errorf("extending PCR %d: %w", pcrIndex, err)
++	}
++	return nil
++}
++
++func toPtr[A any](a A) *A {
++	return &a
++}

packages/by-name/cloud-api-adaptor/0002-set-policy-digest-in-agent-config.patch

@@ -0,0 +1,86 @@
+From 866d001b62bfc6e38424a760e7ef577511afc94c Mon Sep 17 00:00:00 2001
+From: Markus Rudy <[email protected]>
+Date: Wed, 23 Oct 2024 10:50:46 +0200
+Subject: [PATCH] set policy digest in agent config
+
+This allows verifying SetPolicyRequests on the agent side.
+
+Note: this patch needs to be supported by an equivalent patch for the
+Kata agent.
+---
+ src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go |  2 +-
+ src/cloud-api-adaptor/pkg/agent/config.go        | 10 +++++++++-
+ src/cloud-api-adaptor/pkg/util/cloud.go          |  9 +++++++++
+ 3 files changed, 19 insertions(+), 2 deletions(-)
+
+diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go
+index 5a3ab96..0a83683 100644
+--- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go
++++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go
+@@ -239,7 +239,7 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
+ 		logger.Printf("configure agent to use credentials file %s", SrcAuthfilePath)
+ 	}
+
+-	agentConfig, err := agent.CreateConfigFile(authFilePath)
++	agentConfig, err := agent.CreateConfigFile(authFilePath, util.GetPolicyFromAnnotation(req.Annotations))
+ 	if err != nil {
+ 		return nil, fmt.Errorf("creating agent config: %w", err)
+ 	}
+diff --git a/src/cloud-api-adaptor/pkg/agent/config.go b/src/cloud-api-adaptor/pkg/agent/config.go
+index 58bcc83..c0d5b58 100644
+--- a/src/cloud-api-adaptor/pkg/agent/config.go
++++ b/src/cloud-api-adaptor/pkg/agent/config.go
+@@ -1,6 +1,9 @@
+ package agent
+
+ import (
++	"crypto/sha256"
++	"encoding/hex"
++
+ 	"github.com/pelletier/go-toml/v2"
+ )
+
+@@ -13,10 +16,11 @@ const (
+ type agentConfig struct {
+ 	ServerAddr           string `toml:"server_addr"`
+ 	GuestComponentsProcs string `toml:"guest_components_procs"`
++	PolicySHA256Hex      string `toml:"policy_digest_sha256_hex,omitempty"`
+ 	ImageRegistryAuth    string `toml:"image_registry_auth,omitempty"`
+ }
+
+-func CreateConfigFile(authJsonPath string) (string, error) {
++func CreateConfigFile(authJsonPath string, policy []byte) (string, error) {
+ 	var imageRegistryAuth string
+ 	if authJsonPath != "" {
+ 		imageRegistryAuth = "file://" + authJsonPath
+@@ -27,6 +31,10 @@ func CreateConfigFile(authJsonPath string) (string, error) {
+ 		GuestComponentsProcs: GuestComponentsProcs,
+ 		ImageRegistryAuth:    imageRegistryAuth,
+ 	}
++	if policy != nil {
++		digest := sha256.Sum256(policy)
++		config.PolicySHA256Hex = hex.EncodeToString(digest[:])
++	}
+
+ 	bytes, err := toml.Marshal(config)
+ 	if err != nil {
+diff --git a/src/cloud-api-adaptor/pkg/util/cloud.go b/src/cloud-api-adaptor/pkg/util/cloud.go
+index b2ba396..e317495 100644
+--- a/src/cloud-api-adaptor/pkg/util/cloud.go
++++ b/src/cloud-api-adaptor/pkg/util/cloud.go
+@@ -69,6 +69,15 @@ func GetCPUAndMemoryFromAnnotation(annotations map[string]string) (int64, int64)
+ 	return vcpuInt, memoryInt
+ }
+
++func GetPolicyFromAnnotation(annotations map[string]string) []byte {
++	// The policy is already base64-decoded in this annotation map.
++	policy, ok := annotations[hypannotations.Policy]
++	if !ok {
++		return nil
++	}
++	return []byte(policy)
++}
++
+ // Method to check if a string exists in a slice
+ func Contains(slice []string, s string) bool {
+ 	for _, item := range slice {

packages/by-name/cloud-api-adaptor/package.nix

@@ -40,10 +40,23 @@ buildGoModule rec {
     hash = "sha256-5tDG0sEiRAsb259lPui5ntR6DVHDdcXhb04UESJzHhE=";
   };
 
+  patches = [
+    # Make the process-user-data job measure the agent config file into PCR10 (which is otherwise
+    # unused), so that it can be verified in an attestation report.
+    # The CAA attestation story is not decided yet. This patch enables one possible solution for
+    # Contrast.
+    ./0001-measure-agent-config.toml-into-PCR-10.patch
+    # Forward the expected policy hash as part of the agent-config.toml via instance user-data.
+    # Not upstreamable, like the patch above.
+    ./0002-set-policy-digest-in-agent-config.patch
+  ];
+
+  patchFlags = [ "-p3" ];
+
   sourceRoot = "${src.name}/src/cloud-api-adaptor";
 
   proxyVendor = true;
-  vendorHash = "sha256-kqzi7jRF3tQ4/yLkJXfZly4EvVKFb400/WXlN0WjYm8=";
+  vendorHash = "sha256-6FWMh2G5yM0QnhpfLS+fRfP6bpPtuGCeCvCNutog3YU=";
 
   nativeBuildInputs = lib.optional withLibvirt pkg-config;
 

packages/by-name/kata/kata-runtime/0015-agent-read-policy-hash-from-config.patch

@@ -0,0 +1,102 @@
+From 1ba75161e0ac5d404ddf645cdcea848a3a41c2a7 Mon Sep 17 00:00:00 2001
+From: Markus Rudy <[email protected]>
+Date: Thu, 24 Oct 2024 17:06:26 +0200
+Subject: [PATCH] agent: read policy hash from config
+
+---
+ src/agent/Cargo.lock    | 4 ++++
+ src/agent/Cargo.toml    | 1 +
+ src/agent/src/config.rs | 8 ++++++++
+ src/agent/src/policy.rs | 5 ++++-
+ 4 files changed, 17 insertions(+), 1 deletion(-)
+
+diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock
+index 6c9342ddb..1bfc716d3 100644
+--- a/src/agent/Cargo.lock
++++ b/src/agent/Cargo.lock
+@@ -2818,6 +2818,7 @@ dependencies = [
+  "const_format",
+  "derivative",
+  "futures",
++ "hex",
+  "image-rs",
+  "ipnetwork",
+  "kata-sys-util",
+@@ -6439,6 +6440,9 @@ name = "uuid"
+ version = "1.10.0"
+ source = "registry+https://github.com/rust-lang/crates.io-index"
+ checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314"
++dependencies = [
++ "serde",
++]
+
+ [[package]]
+ name = "valuable"
+diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml
+index d5b3db965..44612495a 100644
+--- a/src/agent/Cargo.toml
++++ b/src/agent/Cargo.toml
+@@ -89,6 +89,7 @@ regorus = { version = "0.1.4", default-features = false, features = [
+ sha2 = { version = "0.10.6", optional = true }
+ sev = { version = "2.0.2", default-features = false, features = ["snp"], optional = true }
+ vmm-sys-util = { version = "0.11.0", optional = true }
++hex = "0.4.3"
+
+ [dev-dependencies]
+ tempfile = "3.1.0"
+diff --git a/src/agent/src/config.rs b/src/agent/src/config.rs
+index a83328440..a984f9dcc 100644
+--- a/src/agent/src/config.rs
++++ b/src/agent/src/config.rs
+@@ -131,6 +131,8 @@ pub struct AgentConfig {
+     pub image_policy_file: String,
+     #[cfg(feature = "agent-policy")]
+     pub policy_file: String,
++    #[cfg(feature = "agent-policy")]
++    pub policy_digest_sha256_hex: String,
+ }
+
+ #[derive(Debug, Deserialize)]
+@@ -160,6 +162,8 @@ pub struct AgentConfigBuilder {
+     pub image_policy_file: Option<String>,
+     #[cfg(feature = "agent-policy")]
+     pub policy_file: Option<String>,
++    #[cfg(feature = "agent-policy")]
++    pub policy_digest_sha256_hex: Option<String>,
+ }
+
+ macro_rules! config_override {
+@@ -235,6 +239,8 @@ impl Default for AgentConfig {
+             image_policy_file: String::from(""),
+             #[cfg(feature = "agent-policy")]
+             policy_file: String::from(""),
++            #[cfg(feature = "agent-policy")]
++            policy_digest_sha256_hex: String::from(""),
+         }
+     }
+ }
+@@ -287,6 +293,8 @@ impl FromStr for AgentConfig {
+
+         #[cfg(feature = "agent-policy")]
+         config_override!(agent_config_builder, agent_config, policy_file);
++        #[cfg(feature = "agent-policy")]
++        config_override!(agent_config_builder, agent_config, policy_digest_sha256_hex);
+         Ok(agent_config)
+     }
+ }
+diff --git a/src/agent/src/policy.rs b/src/agent/src/policy.rs
+index 2f1da9ecd..840385fc3 100644
+--- a/src/agent/src/policy.rs
++++ b/src/agent/src/policy.rs
+@@ -198,7 +198,10 @@ impl AgentPolicy {
+ }
+
+ fn verify_policy_digest(policy: &str) -> Result<()> {
+-    if let Ok(expected_digest) = get_tdx_mrconfigid() {
++    if !AGENT_CONFIG.policy_digest_sha256_hex.is_empty() {
++        let expected_digest = hex::decode(&AGENT_CONFIG.policy_digest_sha256_hex)?;
++        verify_sha_256(policy, expected_digest.as_slice())
++    } else if let Ok(expected_digest) = get_tdx_mrconfigid() {
+         info!(sl!(), "policy: TDX MrConfigId ({:?})", expected_digest);
+
+         // The MrConfigId used with TDX is longer than the host-data field used

packages/by-name/kata/kata-runtime/0016-runtime-forward-policy-to-remote-hypervisor.patch

@@ -0,0 +1,21 @@
+From aa0f1beae53bbdf2ebb2a7aad6d7bcb9b7c2ac2f Mon Sep 17 00:00:00 2001
+From: Markus Rudy <[email protected]>
+Date: Fri, 18 Oct 2024 09:58:47 +0200
+Subject: [PATCH] runtime: forward policy to remote hypervisor
+
+---
+ src/runtime/virtcontainers/remote.go | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/src/runtime/virtcontainers/remote.go b/src/runtime/virtcontainers/remote.go
+index 047f09fe8..e95763e44 100644
+--- a/src/runtime/virtcontainers/remote.go
++++ b/src/runtime/virtcontainers/remote.go
+@@ -81,6 +81,7 @@ func (rh *remoteHypervisor) CreateVM(ctx context.Context, id string, network Net
+ 	annotations[hypannotations.DefaultVCPUs] = strconv.FormatUint(uint64(hypervisorConfig.NumVCPUs()), 10)
+ 	annotations[hypannotations.DefaultMemory] = strconv.FormatUint(uint64(hypervisorConfig.MemorySize), 10)
+ 	annotations[hypannotations.Initdata] = hypervisorConfig.Initdata
++	annotations[hypannotations.Policy] = hypervisorConfig.AgentPolicy
+
+ 	req := &pb.CreateVMRequest{
+ 		Id:                   id,

packages/by-name/kata/kata-runtime/package.nix

@@ -90,6 +90,16 @@ buildGoModule rec {
       # The patch is not sufficient for upstream, because it requires the extraRootFs content from
       # our Nix packaging.
       ./0014-tools-don-t-clean-build-root-when-generating-rootfs.patch
+
+      # A peer-pod VM does not have HOSTDATA or MRCONFIGID, so the expected policy hash needs to
+      # be configured differently. This patch adds a policy hash config field to the agent config,
+      # which is passed by the CAA and loaded from user-data.
+      # The upstream plan-of-record is the initdata proposal, which will eventually provide all
+      # podvm configuration in a measurable way. Unfortunately, this proposal has diverged between
+      # Kata and CAA, so we're implementing our own solution here.
+      ./0015-agent-read-policy-hash-from-config.patch
+      # This patch makes the remote hypervisor aware of the workload policy.
+      ./0016-runtime-forward-policy-to-remote-hypervisor.patch
     ];
   };