From 61d83c41466a4f133df08566303e064a5a57d8f0 Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Fri, 18 Oct 2024 12:28:13 +0200 Subject: [PATCH] peer-pods: pass policy hash via userdata --- ...easure-agent-config.toml-into-PCR-10.patch | 126 ++++++++++++++++++ ...02-set-policy-digest-in-agent-config.patch | 86 ++++++++++++ .../by-name/cloud-api-adaptor/package.nix | 15 ++- ...7-agent-read-policy-hash-from-config.patch | 92 +++++++++++++ ...-forward-policy-to-remote-hypervisor.patch | 21 +++ .../by-name/kata/kata-runtime/package.nix | 10 ++ 6 files changed, 349 insertions(+), 1 deletion(-) create mode 100644 packages/by-name/cloud-api-adaptor/0001-measure-agent-config.toml-into-PCR-10.patch create mode 100644 packages/by-name/cloud-api-adaptor/0002-set-policy-digest-in-agent-config.patch create mode 100644 packages/by-name/kata/kata-runtime/0017-agent-read-policy-hash-from-config.patch create mode 100644 packages/by-name/kata/kata-runtime/0018-runtime-forward-policy-to-remote-hypervisor.patch diff --git a/packages/by-name/cloud-api-adaptor/0001-measure-agent-config.toml-into-PCR-10.patch b/packages/by-name/cloud-api-adaptor/0001-measure-agent-config.toml-into-PCR-10.patch new file mode 100644 index 0000000000..39038e6617 --- /dev/null +++ b/packages/by-name/cloud-api-adaptor/0001-measure-agent-config.toml-into-PCR-10.patch @@ -0,0 +1,126 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Markus Rudy +Date: Wed, 23 Oct 2024 10:50:46 +0200 +Subject: [PATCH] measure agent-config.toml into PCR 10 + +The agent config is security critical and needs to be measurable. It +would be nice to measure the daemon config, too, but it contains +hard-to-predict networking configuration (such as the node IP). + +This commit removes the files not relevant for Contrast out of an +abundance of caution. +--- + src/cloud-api-adaptor/go.mod | 5 +++- + src/cloud-api-adaptor/go.sum | 2 ++ + .../pkg/userdata/provision.go | 30 ++++++++++++++++--- + 3 files changed, 32 insertions(+), 5 deletions(-) + +diff --git a/src/cloud-api-adaptor/go.mod b/src/cloud-api-adaptor/go.mod +index bd419f65c99429b3fd20f850509fb0223d82c41d..011870a713eaee698e577e2b55bc05eed37f2104 100644 +--- a/src/cloud-api-adaptor/go.mod ++++ b/src/cloud-api-adaptor/go.mod +@@ -1,6 +1,8 @@ + module github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor + +-go 1.21 ++go 1.22 ++ ++toolchain go1.23.2 + + require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 +@@ -52,6 +54,7 @@ require ( + github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f + github.com/docker/docker v25.0.5+incompatible + github.com/golang-jwt/jwt/v5 v5.2.1 ++ github.com/google/go-tpm v0.9.1 + github.com/moby/sys/mountinfo v0.7.1 + github.com/pelletier/go-toml/v2 v2.1.0 + github.com/sirupsen/logrus v1.9.3 +diff --git a/src/cloud-api-adaptor/go.sum b/src/cloud-api-adaptor/go.sum +index 0dd05c2182c891213db3c6920c82702bea6e1f3f..1ffa1f8366a60de6743314fab67e2dfe9b73d266 100644 +--- a/src/cloud-api-adaptor/go.sum ++++ b/src/cloud-api-adaptor/go.sum +@@ -322,6 +322,8 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN + github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= + github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= + github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= ++github.com/google/go-tpm v0.9.1 h1:0pGc4X//bAlmZzMKf8iz6IsDo1nYTbYJ6FZN/rg4zdM= ++github.com/google/go-tpm v0.9.1/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= + github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= + github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= + github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +diff --git a/src/cloud-api-adaptor/pkg/userdata/provision.go b/src/cloud-api-adaptor/pkg/userdata/provision.go +index 5c3b6caf560d4c2fc97d3bd5fc23f1aef78750b3..e74169939e3526acfda17228231ad3e27376aeef 100644 +--- a/src/cloud-api-adaptor/pkg/userdata/provision.go ++++ b/src/cloud-api-adaptor/pkg/userdata/provision.go +@@ -2,6 +2,7 @@ package userdata + + import ( + "context" ++ "crypto/sha256" + "fmt" + "log" + "os" +@@ -12,6 +13,8 @@ import ( + "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/aws" + "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/azure" + "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/docker" ++ "github.com/google/go-tpm/legacy/tpm2" ++ "github.com/google/go-tpm/tpmutil" + "gopkg.in/yaml.v2" + ) + +@@ -162,6 +165,7 @@ func findConfigEntry(path string, cc *CloudConfig) []byte { + type entry struct { + path string + optional bool ++ pcrIndex *int + } + + func (f *entry) writeFile(cc *CloudConfig) error { +@@ -179,6 +183,10 @@ func (f *entry) writeFile(cc *CloudConfig) error { + return fmt.Errorf("failed to create directory: %w", err) + } + ++ if f.pcrIndex != nil { ++ extendPCR(*f.pcrIndex, bytes) ++ } ++ + err = os.WriteFile(f.path, bytes, 0644) + if err != nil { + return fmt.Errorf("failed to write file: %w", err) +@@ -189,11 +197,8 @@ func (f *entry) writeFile(cc *CloudConfig) error { + + func processCloudConfig(cfg *Config, cc *CloudConfig) error { + entries := []entry{ +- {path: cfg.paths.agentConfig, optional: false}, ++ {path: cfg.paths.agentConfig, optional: false, pcrIndex: toPtr(10)}, + {path: cfg.paths.daemonConfig, optional: false}, +- {path: cfg.paths.aaConfig, optional: true}, +- {path: cfg.paths.cdhConfig, optional: true}, +- {path: cfg.paths.authJson, optional: true}, + } + + for _, e := range entries { +@@ -228,3 +233,20 @@ func ProvisionFiles(cfg *Config) error { + + return nil + } ++ ++func extendPCR(pcrIndex int, data []byte) error { ++ digest := sha256.Sum256(data) ++ ++ handle, err := tpm2.OpenTPM() ++ if err != nil { ++ return fmt.Errorf("opening TPM device: %w", err) ++ } ++ if err := tpm2.PCRExtend(handle, tpmutil.Handle(pcrIndex), tpm2.AlgSHA256, digest[:], ""); err != nil { ++ return fmt.Errorf("extending PCR %d: %w", pcrIndex, err) ++ } ++ return nil ++} ++ ++func toPtr[A any](a A) *A { ++ return &a ++} diff --git a/packages/by-name/cloud-api-adaptor/0002-set-policy-digest-in-agent-config.patch b/packages/by-name/cloud-api-adaptor/0002-set-policy-digest-in-agent-config.patch new file mode 100644 index 0000000000..33f0cd5bdd --- /dev/null +++ b/packages/by-name/cloud-api-adaptor/0002-set-policy-digest-in-agent-config.patch @@ -0,0 +1,86 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Markus Rudy +Date: Wed, 23 Oct 2024 10:50:46 +0200 +Subject: [PATCH] set policy digest in agent config + +This allows verifying SetPolicyRequests on the agent side. + +Note: this patch needs to be supported by an equivalent patch for the +Kata agent. +--- + src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go | 2 +- + src/cloud-api-adaptor/pkg/agent/config.go | 10 +++++++++- + src/cloud-api-adaptor/pkg/util/cloud.go | 9 +++++++++ + 3 files changed, 19 insertions(+), 2 deletions(-) + +diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go +index 5a3ab96697a2f83f5499572eac76942b383c5a82..0a83683a156b8459a5706f9df12150e5114f3792 100644 +--- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go ++++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go +@@ -239,7 +239,7 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r + logger.Printf("configure agent to use credentials file %s", SrcAuthfilePath) + } + +- agentConfig, err := agent.CreateConfigFile(authFilePath) ++ agentConfig, err := agent.CreateConfigFile(authFilePath, util.GetPolicyFromAnnotation(req.Annotations)) + if err != nil { + return nil, fmt.Errorf("creating agent config: %w", err) + } +diff --git a/src/cloud-api-adaptor/pkg/agent/config.go b/src/cloud-api-adaptor/pkg/agent/config.go +index 58bcc83435d62eaafe7d5972df5772741141d31d..c0d5b58dfc8cc46bc2e7721b1d7436d194058ea7 100644 +--- a/src/cloud-api-adaptor/pkg/agent/config.go ++++ b/src/cloud-api-adaptor/pkg/agent/config.go +@@ -1,6 +1,9 @@ + package agent + + import ( ++ "crypto/sha256" ++ "encoding/hex" ++ + "github.com/pelletier/go-toml/v2" + ) + +@@ -13,10 +16,11 @@ const ( + type agentConfig struct { + ServerAddr string `toml:"server_addr"` + GuestComponentsProcs string `toml:"guest_components_procs"` ++ PolicySHA256Hex string `toml:"policy_digest_sha256_hex,omitempty"` + ImageRegistryAuth string `toml:"image_registry_auth,omitempty"` + } + +-func CreateConfigFile(authJsonPath string) (string, error) { ++func CreateConfigFile(authJsonPath string, policy []byte) (string, error) { + var imageRegistryAuth string + if authJsonPath != "" { + imageRegistryAuth = "file://" + authJsonPath +@@ -27,6 +31,10 @@ func CreateConfigFile(authJsonPath string) (string, error) { + GuestComponentsProcs: GuestComponentsProcs, + ImageRegistryAuth: imageRegistryAuth, + } ++ if policy != nil { ++ digest := sha256.Sum256(policy) ++ config.PolicySHA256Hex = hex.EncodeToString(digest[:]) ++ } + + bytes, err := toml.Marshal(config) + if err != nil { +diff --git a/src/cloud-api-adaptor/pkg/util/cloud.go b/src/cloud-api-adaptor/pkg/util/cloud.go +index b2ba396af486aafd5030cba90a62037470f466d7..e31749538a04b24a367320fdcb60db7614156c71 100644 +--- a/src/cloud-api-adaptor/pkg/util/cloud.go ++++ b/src/cloud-api-adaptor/pkg/util/cloud.go +@@ -69,6 +69,15 @@ func GetCPUAndMemoryFromAnnotation(annotations map[string]string) (int64, int64) + return vcpuInt, memoryInt + } + ++func GetPolicyFromAnnotation(annotations map[string]string) []byte { ++ // The policy is already base64-decoded in this annotation map. ++ policy, ok := annotations[hypannotations.Policy] ++ if !ok { ++ return nil ++ } ++ return []byte(policy) ++} ++ + // Method to check if a string exists in a slice + func Contains(slice []string, s string) bool { + for _, item := range slice { diff --git a/packages/by-name/cloud-api-adaptor/package.nix b/packages/by-name/cloud-api-adaptor/package.nix index ab008d7ba6..47e0e775d9 100644 --- a/packages/by-name/cloud-api-adaptor/package.nix +++ b/packages/by-name/cloud-api-adaptor/package.nix @@ -40,10 +40,23 @@ buildGoModule rec { hash = "sha256-5tDG0sEiRAsb259lPui5ntR6DVHDdcXhb04UESJzHhE="; }; + patches = [ + # Make the process-user-data job measure the agent config file into PCR10 (which is otherwise + # unused), so that it can be verified in an attestation report. + # The CAA attestation story is not decided yet. This patch enables one possible solution for + # Contrast. + ./0001-measure-agent-config.toml-into-PCR-10.patch + # Forward the expected policy hash as part of the agent-config.toml via instance user-data. + # Not upstreamable, like the patch above. + ./0002-set-policy-digest-in-agent-config.patch + ]; + + patchFlags = [ "-p3" ]; + sourceRoot = "${src.name}/src/cloud-api-adaptor"; proxyVendor = true; - vendorHash = "sha256-kqzi7jRF3tQ4/yLkJXfZly4EvVKFb400/WXlN0WjYm8="; + vendorHash = "sha256-6FWMh2G5yM0QnhpfLS+fRfP6bpPtuGCeCvCNutog3YU="; nativeBuildInputs = lib.optional withLibvirt pkg-config; diff --git a/packages/by-name/kata/kata-runtime/0017-agent-read-policy-hash-from-config.patch b/packages/by-name/kata/kata-runtime/0017-agent-read-policy-hash-from-config.patch new file mode 100644 index 0000000000..70ce196b56 --- /dev/null +++ b/packages/by-name/kata/kata-runtime/0017-agent-read-policy-hash-from-config.patch @@ -0,0 +1,92 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Markus Rudy +Date: Thu, 24 Oct 2024 17:06:26 +0200 +Subject: [PATCH] agent: read policy hash from config + +--- + src/agent/Cargo.lock | 1 + + src/agent/Cargo.toml | 1 + + src/agent/src/config.rs | 8 ++++++++ + src/agent/src/policy.rs | 5 ++++- + 4 files changed, 14 insertions(+), 1 deletion(-) + +diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock +index eae6db554824b9de0d9694d583d0b05d610c5d9a..731123e5a33dbf60108f1fdf188be40d63f94085 100644 +--- a/src/agent/Cargo.lock ++++ b/src/agent/Cargo.lock +@@ -2818,6 +2818,7 @@ dependencies = [ + "const_format", + "derivative", + "futures", ++ "hex", + "image-rs", + "ipnetwork", + "kata-sys-util", +diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml +index d5b3db965fe75cbccc182825a4115bdc57a9705b..44612495a9a7891441ae1bded737edec718e79e1 100644 +--- a/src/agent/Cargo.toml ++++ b/src/agent/Cargo.toml +@@ -89,6 +89,7 @@ regorus = { version = "0.1.4", default-features = false, features = [ + sha2 = { version = "0.10.6", optional = true } + sev = { version = "2.0.2", default-features = false, features = ["snp"], optional = true } + vmm-sys-util = { version = "0.11.0", optional = true } ++hex = "0.4.3" + + [dev-dependencies] + tempfile = "3.1.0" +diff --git a/src/agent/src/config.rs b/src/agent/src/config.rs +index 0ae3a94fbdcf8fb8c462d27939207cd600228a73..37a32017a099d0a9526729c5e92528f218d05c47 100644 +--- a/src/agent/src/config.rs ++++ b/src/agent/src/config.rs +@@ -131,6 +131,8 @@ pub struct AgentConfig { + pub image_policy_file: String, + #[cfg(feature = "agent-policy")] + pub policy_file: String, ++ #[cfg(feature = "agent-policy")] ++ pub policy_digest_sha256_hex: String, + } + + #[derive(Debug, Deserialize)] +@@ -160,6 +162,8 @@ pub struct AgentConfigBuilder { + pub image_policy_file: Option, + #[cfg(feature = "agent-policy")] + pub policy_file: Option, ++ #[cfg(feature = "agent-policy")] ++ pub policy_digest_sha256_hex: Option, + } + + macro_rules! config_override { +@@ -235,6 +239,8 @@ impl Default for AgentConfig { + image_policy_file: String::from(""), + #[cfg(feature = "agent-policy")] + policy_file: String::from(""), ++ #[cfg(feature = "agent-policy")] ++ policy_digest_sha256_hex: String::from(""), + } + } + } +@@ -287,6 +293,8 @@ impl FromStr for AgentConfig { + + #[cfg(feature = "agent-policy")] + config_override!(agent_config_builder, agent_config, policy_file); ++ #[cfg(feature = "agent-policy")] ++ config_override!(agent_config_builder, agent_config, policy_digest_sha256_hex); + Ok(agent_config) + } + } +diff --git a/src/agent/src/policy.rs b/src/agent/src/policy.rs +index 2f1da9ecd0d0ee1be06218d5bc9e58cd93defa8c..840385fc324fd29749bbff0c9642e6925b70d429 100644 +--- a/src/agent/src/policy.rs ++++ b/src/agent/src/policy.rs +@@ -198,7 +198,10 @@ impl AgentPolicy { + } + + fn verify_policy_digest(policy: &str) -> Result<()> { +- if let Ok(expected_digest) = get_tdx_mrconfigid() { ++ if !AGENT_CONFIG.policy_digest_sha256_hex.is_empty() { ++ let expected_digest = hex::decode(&AGENT_CONFIG.policy_digest_sha256_hex)?; ++ verify_sha_256(policy, expected_digest.as_slice()) ++ } else if let Ok(expected_digest) = get_tdx_mrconfigid() { + info!(sl!(), "policy: TDX MrConfigId ({:?})", expected_digest); + + // The MrConfigId used with TDX is longer than the host-data field used diff --git a/packages/by-name/kata/kata-runtime/0018-runtime-forward-policy-to-remote-hypervisor.patch b/packages/by-name/kata/kata-runtime/0018-runtime-forward-policy-to-remote-hypervisor.patch new file mode 100644 index 0000000000..ee21b1ed5d --- /dev/null +++ b/packages/by-name/kata/kata-runtime/0018-runtime-forward-policy-to-remote-hypervisor.patch @@ -0,0 +1,21 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Markus Rudy +Date: Fri, 18 Oct 2024 09:58:47 +0200 +Subject: [PATCH] runtime: forward policy to remote hypervisor + +--- + src/runtime/virtcontainers/remote.go | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/src/runtime/virtcontainers/remote.go b/src/runtime/virtcontainers/remote.go +index 047f09fe8c2db38469440508996b8359cfc8fcba..e95763e44c9bc3c167bbafa615069ef1b0af41a2 100644 +--- a/src/runtime/virtcontainers/remote.go ++++ b/src/runtime/virtcontainers/remote.go +@@ -81,6 +81,7 @@ func (rh *remoteHypervisor) CreateVM(ctx context.Context, id string, network Net + annotations[hypannotations.DefaultVCPUs] = strconv.FormatUint(uint64(hypervisorConfig.NumVCPUs()), 10) + annotations[hypannotations.DefaultMemory] = strconv.FormatUint(uint64(hypervisorConfig.MemorySize), 10) + annotations[hypannotations.Initdata] = hypervisorConfig.Initdata ++ annotations[hypannotations.Policy] = hypervisorConfig.AgentPolicy + + req := &pb.CreateVMRequest{ + Id: id, diff --git a/packages/by-name/kata/kata-runtime/package.nix b/packages/by-name/kata/kata-runtime/package.nix index 087d0aa278..38b4edc762 100644 --- a/packages/by-name/kata/kata-runtime/package.nix +++ b/packages/by-name/kata/kata-runtime/package.nix @@ -93,6 +93,16 @@ buildGoModule rec { ./0014-kata-sys-util-remove-obsolete-cgroups-dependency.patch ./0015-kata-sys-util-move-json-parsing-to-protocols-crate.patch ./0016-protocols-only-build-RLimit-impls-on-Linux.patch + + # A peer-pod VM does not have HOSTDATA or MRCONFIGID, so the expected policy hash needs to + # be configured differently. This patch adds a policy hash config field to the agent config, + # which is passed by the CAA and loaded from user-data. + # The upstream plan-of-record is the initdata proposal, which will eventually provide all + # podvm configuration in a measurable way. Unfortunately, this proposal has diverged between + # Kata and CAA, so we're implementing our own solution here. + ./0017-agent-read-policy-hash-from-config.patch + # This patch makes the remote hypervisor aware of the workload policy. + ./0018-runtime-forward-policy-to-remote-hypervisor.patch ]; };