From 9d8d5cd723e2cf1155f6d650e05e8f6bd5a2ef1a Mon Sep 17 00:00:00 2001 From: Leonard Cohnen Date: Tue, 29 Oct 2024 16:06:31 +0100 Subject: [PATCH] wip: introcude AKSPEERSNP platform --- cli/cmd/generate.go | 12 ++++-- cli/genpolicy/config.go | 2 +- cli/genpolicy/genpolicy.go | 1 + e2e/internal/contrasttest/contrasttest.go | 12 +++++- e2e/openssl/openssl_test.go | 6 ++- internal/kuberesource/mutators.go | 2 +- internal/kuberesource/parts.go | 49 ++++++++++++++++++++++- internal/kuberesource/parts_test.go | 2 +- internal/kuberesource/sets.go | 16 +++++++- internal/manifest/runtimehandler.go | 4 ++ internal/platforms/platforms.go | 6 +++ packages/by-name/contrast/package.nix | 5 +++ 12 files changed, 105 insertions(+), 12 deletions(-) diff --git a/cli/cmd/generate.go b/cli/cmd/generate.go index a3db887b04..f1deb8f4d2 100644 --- a/cli/cmd/generate.go +++ b/cli/cmd/generate.go @@ -217,12 +217,16 @@ func findGenerateTargets(args []string, logger *slog.Logger) ([]string, error) { return nil, fmt.Errorf("no .yml/.yaml files found") } - paths = filterNonCoCoRuntime("contrast-cc", paths, logger) - if len(paths) == 0 { - return nil, fmt.Errorf("no .yml/.yaml files with 'contrast-cc' runtime found") + contrastPaths := filterNonCoCoRuntime("contrast-cc", paths, logger) + if len(contrastPaths) != 0 { + return contrastPaths, nil + } + peerPaths := filterNonCoCoRuntime("kata-remote", paths, logger) + if len(peerPaths) != 0 { + return peerPaths, nil } - return paths, nil + return nil, fmt.Errorf("no .yml/.yaml files with 'contrast-cc' or 'kata-remote' runtime found") } func filterNonCoCoRuntime(runtimeClassNamePrefix string, paths []string, logger *slog.Logger) []string { diff --git a/cli/genpolicy/config.go b/cli/genpolicy/config.go index d48ebda7bd..125448af2a 100644 --- a/cli/genpolicy/config.go +++ b/cli/genpolicy/config.go @@ -44,7 +44,7 @@ func NewConfig(platform platforms.Platform) (*Config, error) { Settings: aksSettings, Bin: aksGenpolicyBin, }, nil - case platforms.K3sQEMUSNP, platforms.K3sQEMUTDX, platforms.RKE2QEMUTDX: + case platforms.K3sQEMUSNP, platforms.K3sQEMUTDX, platforms.RKE2QEMUTDX, platforms.AKSPEERSNP: return &Config{ Rules: kataRules, Settings: kataSettings, diff --git a/cli/genpolicy/genpolicy.go b/cli/genpolicy/genpolicy.go index 8fd6cbc477..aabbb59be6 100644 --- a/cli/genpolicy/genpolicy.go +++ b/cli/genpolicy/genpolicy.go @@ -54,6 +54,7 @@ func New(rulesPath, settingsPath, cachePath string, bin []byte) (*Runner, error) func (r *Runner) Run(ctx context.Context, yamlPath string, logger *slog.Logger) error { args := []string{ "--runtime-class-names=contrast-cc", + "--runtime-class-names=kata-remote", "--rego-rules-path=" + r.rulesPath, "--json-settings-path=" + r.settingsPath, "--layers-cache-file-path=" + r.cachePath, diff --git a/e2e/internal/contrasttest/contrasttest.go b/e2e/internal/contrasttest/contrasttest.go index 2718f58fc0..c429360afa 100644 --- a/e2e/internal/contrasttest/contrasttest.go +++ b/e2e/internal/contrasttest/contrasttest.go @@ -146,7 +146,11 @@ func (ct *ContrastTest) Init(t *testing.T, resources []any) { require.NoError(err) require.NoError(os.WriteFile(path.Join(ct.WorkDir, "resources.yml"), buf, 0o644)) - ct.installRuntime(t) + if ct.Platform == platforms.AKSPEERSNP { + t.Log("Skipping runtime installation for AKS-PEER-SNP") + } else { + ct.installRuntime(t) + } } // Generate runs the contrast generate command. @@ -196,7 +200,7 @@ func (ct *ContrastTest) patchReferenceValues(t *testing.T, platform platforms.Pl SNPVersion: toPtr(manifest.SVN(255)), MicrocodeVersion: toPtr(manifest.SVN(255)), } - case platforms.K3sQEMUSNP: + case platforms.K3sQEMUSNP, platforms.AKSPEERSNP: // The generate command doesn't fill in all required fields when // generating a manifest for baremetal SNP. Do that now. for i, snp := range m.ReferenceValues.SNP { @@ -214,6 +218,8 @@ func (ct *ContrastTest) patchReferenceValues(t *testing.T, platform platforms.Pl tdx.MrSeam = manifest.HexString("1cc6a17ab799e9a693fac7536be61c12ee1e0fabada82d0c999e08ccee2aa86de77b0870f558c570e7ffe55d6d47fa04") m.ReferenceValues.TDX[i] = tdx } + default: + require.NoError(t, fmt.Errorf("unsupported platform %s", platform)) } manifestBytes, err = json.Marshal(m) @@ -367,6 +373,8 @@ func (ct *ContrastTest) FactorPlatformTimeout(timeout time.Duration) time.Durati return timeout case platforms.K3sQEMUSNP, platforms.K3sQEMUTDX, platforms.RKE2QEMUTDX: return 2 * timeout + case platforms.AKSPEERSNP: + return 3 * timeout default: return timeout } diff --git a/e2e/openssl/openssl_test.go b/e2e/openssl/openssl_test.go index a69728b010..bd7e5e31a0 100644 --- a/e2e/openssl/openssl_test.go +++ b/e2e/openssl/openssl_test.go @@ -50,7 +50,7 @@ func TestOpenSSL(t *testing.T) { require.NoError(t, err) resources := kuberesource.OpenSSL() - coordinator := kuberesource.CoordinatorBundle() + coordinator := kuberesource.CoordinatorBundleWith(true) resources = append(resources, coordinator...) @@ -194,6 +194,10 @@ func TestOpenSSL(t *testing.T) { } t.Run("coordinator recovery", func(t *testing.T) { + if platform == platforms.AKSPEERSNP { + t.Skip("coordinator recovery test is not supported on AKSPEERSNP") + } + require := require.New(t) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) // Already long timeout, not using ct.FactorPlatformTimeout. diff --git a/internal/kuberesource/mutators.go b/internal/kuberesource/mutators.go index 63ad4a1236..d27612e5bf 100644 --- a/internal/kuberesource/mutators.go +++ b/internal/kuberesource/mutators.go @@ -36,7 +36,7 @@ func AddInitializer( if meta.Annotations[skipInitializerAnnotationKey] == "true" { return meta, spec } - if spec.RuntimeClassName == nil || !strings.HasPrefix(*spec.RuntimeClassName, "contrast-cc") { + if spec.RuntimeClassName == nil || (!strings.HasPrefix(*spec.RuntimeClassName, "contrast-cc") && !strings.HasPrefix(*spec.RuntimeClassName, "kata-remote")) { return meta, spec } diff --git a/internal/kuberesource/parts.go b/internal/kuberesource/parts.go index 056aadb958..ac337f0b29 100644 --- a/internal/kuberesource/parts.go +++ b/internal/kuberesource/parts.go @@ -136,6 +136,9 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata:latest" snapshotter = nydusSnapshotter snapshotterVolumes = nydusSnapshotterVolumes + case platforms.AKSPEERSNP: + // Node installer for SNP peer pods is currently not implemented. Wait for https://github.com/edgelesssys/contrast/pull/959. + nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-microsoft:latest" default: return nil, fmt.Errorf("unsupported platform %q", platform) } @@ -282,7 +285,7 @@ type CoordinatorConfig struct { } // Coordinator constructs a new CoordinatorConfig. -func Coordinator(namespace string) *CoordinatorConfig { +func Coordinator(namespace string, withoutState bool) *CoordinatorConfig { c := StatefulSet("coordinator", namespace). WithSpec(StatefulSetSpec(). WithReplicas(1). @@ -341,6 +344,50 @@ func Coordinator(namespace string) *CoordinatorConfig { ), ) + if withoutState { + c = StatefulSet("coordinator", namespace). + WithSpec(StatefulSetSpec(). + WithReplicas(1). + WithServiceName("coordinator"). + WithSelector(LabelSelector(). + WithMatchLabels(map[string]string{"app.kubernetes.io/name": "coordinator"}), + ). + WithTemplate(PodTemplateSpec(). + WithLabels(map[string]string{"app.kubernetes.io/name": "coordinator"}). + WithAnnotations(map[string]string{"contrast.edgeless.systems/pod-role": "coordinator"}). + WithSpec(PodSpec(). + WithContainers( + Container(). + WithName("coordinator"). + WithImage("ghcr.io/edgelesssys/contrast/coordinator:latest"). + WithSecurityContext(SecurityContext(). + WithCapabilities(applycorev1.Capabilities(). + WithAdd("SYS_ADMIN"), + ), + ). + WithPorts( + ContainerPort(). + WithName("userapi"). + WithContainerPort(1313), + ContainerPort(). + WithName("meshapi"). + WithContainerPort(7777), + ). + WithReadinessProbe(Probe(). + WithInitialDelaySeconds(1). + WithPeriodSeconds(5). + WithTCPSocket(TCPSocketAction(). + WithPort(intstr.FromInt(1313))), + ). + WithResources(ResourceRequirements(). + WithMemoryLimitAndRequest(100), + ), + ), + ), + ), + ) + } + return &CoordinatorConfig{c} } diff --git a/internal/kuberesource/parts_test.go b/internal/kuberesource/parts_test.go index 7793b422a5..7f68a75520 100644 --- a/internal/kuberesource/parts_test.go +++ b/internal/kuberesource/parts_test.go @@ -24,7 +24,7 @@ func TestNewPortForwarder(t *testing.T) { func TestCoordinator(t *testing.T) { require := require.New(t) - b, err := EncodeResources(Coordinator("default")) + b, err := EncodeResources(Coordinator("default", false)) require.NoError(err) t.Log("\n" + string(b)) } diff --git a/internal/kuberesource/sets.go b/internal/kuberesource/sets.go index 4c9010779a..98f3a5e848 100644 --- a/internal/kuberesource/sets.go +++ b/internal/kuberesource/sets.go @@ -17,7 +17,21 @@ import ( // CoordinatorBundle returns the Coordinator and a matching Service. func CoordinatorBundle() []any { - coordinatorSfSets := Coordinator("").StatefulSetApplyConfiguration + coordinatorSfSets := Coordinator("", false).StatefulSetApplyConfiguration + coordinatorService := ServiceForStatefulSet(coordinatorSfSets). + WithAnnotations(map[string]string{exposeServiceAnnotation: "true"}) + + resources := []any{ + coordinatorSfSets, + coordinatorService, + } + + return resources +} + +// CoordinatorBundleWith returns the Coordinator and a matching Service. +func CoordinatorBundleWith(withoutState bool) []any { + coordinatorSfSets := Coordinator("", withoutState).StatefulSetApplyConfiguration coordinatorService := ServiceForStatefulSet(coordinatorSfSets). WithAnnotations(map[string]string{exposeServiceAnnotation: "true"}) diff --git a/internal/manifest/runtimehandler.go b/internal/manifest/runtimehandler.go index de8134ac89..89eec08ab2 100644 --- a/internal/manifest/runtimehandler.go +++ b/internal/manifest/runtimehandler.go @@ -17,6 +17,10 @@ func RuntimeHandler(platform platforms.Platform) (string, error) { return "", fmt.Errorf("unmarshal embedded reference values mapping: %w", err) } + if platform == platforms.AKSPEERSNP { + return "kata-remote", nil + } + for runtimeHandler := range mapping { p, err := platformFromHandler(runtimeHandler) if err != nil { diff --git a/internal/platforms/platforms.go b/internal/platforms/platforms.go index 92966870c0..57ad7fc946 100644 --- a/internal/platforms/platforms.go +++ b/internal/platforms/platforms.go @@ -24,6 +24,8 @@ const ( K3sQEMUSNP // RKE2QEMUTDX represents a deployment with QEMU on bare-metal TDX RKE2. RKE2QEMUTDX + // AKSPEERSNP represents a deployment with PeerPod on SEV-SNP AKS. + AKSPEERSNP ) // All returns a list of all available platforms. @@ -51,6 +53,8 @@ func (p Platform) String() string { return "K3s-QEMU-SNP" case RKE2QEMUTDX: return "RKE2-QEMU-TDX" + case AKSPEERSNP: + return "AKS-PEER-SNP" default: return "Unknown" } @@ -67,6 +71,8 @@ func FromString(s string) (Platform, error) { return K3sQEMUSNP, nil case "rke2-qemu-tdx": return RKE2QEMUTDX, nil + case "aks-peer-snp": + return AKSPEERSNP, nil default: return Unknown, fmt.Errorf("unknown platform: %s", s) } diff --git a/packages/by-name/contrast/package.nix b/packages/by-name/contrast/package.nix index 1326394eaa..a2925b5040 100644 --- a/packages/by-name/contrast/package.nix +++ b/packages/by-name/contrast/package.nix @@ -53,6 +53,8 @@ let k3s-qemu-tdx-handler = runtimeHandler "k3s-qemu-tdx" kata.contrast-node-installer-image.runtimeHash; rke2-qemu-tdx-handler = runtimeHandler "rke2-qemu-tdx" kata.contrast-node-installer-image.runtimeHash; k3s-qemu-snp-handler = runtimeHandler "k3s-qemu-snp" kata.contrast-node-installer-image.runtimeHash; + # We currently don't have our own node installer. Wait for https://github.com/edgelesssys/contrast/pull/959. + aks-peer-snp-handler = runtimeHandler "aks-peer-snp" kata.contrast-node-installer-image.runtimeHash; aksRefVals = { snp = [ @@ -129,6 +131,9 @@ let "${k3s-qemu-tdx-handler}" = tdxRefVals; "${rke2-qemu-tdx-handler}" = tdxRefVals; "${k3s-qemu-snp-handler}" = snpRefVals; + # TODO(@3u13r): We don't currently have an implemented attestation story for SNP peer pods. + # Use the snpRefVals as a placeholder for now. + "${aks-peer-snp-handler}" = snpRefVals; } );