From 1b2f5a3d8dd75eb634fa61582e9356da7b184249 Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Wed, 17 Apr 2024 15:29:42 +0200 Subject: [PATCH 1/7] node-installer: diff patched containerd config with actual file contents --- node-installer/node-installer.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/node-installer/node-installer.go b/node-installer/node-installer.go index 55f6fe941b..8ea1e99ee4 100644 --- a/node-installer/node-installer.go +++ b/node-installer/node-installer.go @@ -115,14 +115,10 @@ func containerdRuntimeConfig(basePath, configPath string) error { } func patchContainerdConfig(runtimeName, basePath, configPath string) error { - existing, err := parseExistingContainerdConfig(configPath) + existingRaw, existing, err := parseExistingContainerdConfig(configPath) if err != nil { existing = constants.ContainerdBaseConfig() } - existingRaw, err := toml.Marshal(existing) - if err != nil { - return err - } // Add tardev snapshotter if existing.ProxyPlugins == nil { @@ -142,6 +138,7 @@ func patchContainerdConfig(runtimeName, basePath, configPath string) error { } if slices.Equal(existingRaw, rawConfig) { + fmt.Println("Containerd config already up-to-date. No changes needed.") return nil } @@ -149,18 +146,18 @@ func patchContainerdConfig(runtimeName, basePath, configPath string) error { return os.WriteFile(configPath, rawConfig, os.ModePerm) } -func parseExistingContainerdConfig(path string) (config.ContainerdConfig, error) { +func parseExistingContainerdConfig(path string) ([]byte, config.ContainerdConfig, error) { configData, err := os.ReadFile(path) if err != nil { - return config.ContainerdConfig{}, err + return nil, config.ContainerdConfig{}, err } var cfg config.ContainerdConfig if err := toml.Unmarshal(configData, &cfg); err != nil { - return config.ContainerdConfig{}, err + return nil, config.ContainerdConfig{}, err } - return cfg, nil + return configData, cfg, nil } func restartHostContainerd(containerdConfigPath string) error { @@ -189,6 +186,7 @@ func restartHostContainerd(containerdConfigPath string) error { } fmt.Printf("containerd start time: %s\n", startTime.Format(time.RFC3339)) + fmt.Printf("config mtime: %s\n", configMtime.Format(time.RFC3339)) if startTime.After(configMtime) { fmt.Println("containerd already running with the newest config") return nil From c7c03bf95610be8b913808cc8851599fc11a55ba Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Wed, 17 Apr 2024 13:36:36 +0200 Subject: [PATCH 2/7] cli: add runtime subcommand --- cli/cmd/common.go | 4 ++++ cli/cmd/runtime.go | 31 +++++++++++++++++++++++---- cli/main.go | 1 + packages/by-name/contrast/package.nix | 1 + 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/cli/cmd/common.go b/cli/cmd/common.go index f89cb3c6cd..05efd956ba 100644 --- a/cli/cmd/common.go +++ b/cli/cmd/common.go @@ -25,6 +25,10 @@ var ( defaultGenpolicySettings []byte //go:embed assets/genpolicy-rules.rego defaultRules []byte + // DefaultCoordinatorPolicyHash is derived from the coordinator release candidate and injected at release build time. + // + // It is intentionally left empty for dev builds. + DefaultCoordinatorPolicyHash = "" ) func cachedir(subdir string) (string, error) { diff --git a/cli/cmd/runtime.go b/cli/cmd/runtime.go index f3828b06f6..4005b66c67 100644 --- a/cli/cmd/runtime.go +++ b/cli/cmd/runtime.go @@ -1,6 +1,29 @@ package cmd -// DefaultCoordinatorPolicyHash is derived from the coordinator release candidate and injected at release build time. -// -// It is intentionally left empty for dev builds. -var DefaultCoordinatorPolicyHash = "" +import ( + "github.com/spf13/cobra" +) + +// This value is injected at build time. +var runtimeHandler = "contrast-cc" + +// NewRuntimeCmd creates the contrast runtime subcommand. +func NewRuntimeCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "runtime", + Short: "Prints the runtimeClassName", + Long: `Prints runtimeClassName used by Contrast. + +Contrast uses a custom container runtime, where every pod is a confidential +virtual machine. Pod specs of workloads running on Contrast must +have the runtimeClassName set to the value returned by this command. + `, + Run: runRuntime, + } + + return cmd +} + +func runRuntime(cmd *cobra.Command, _ []string) { + cmd.Println(runtimeHandler) +} diff --git a/cli/main.go b/cli/main.go index 8796c57258..ce5d45482a 100644 --- a/cli/main.go +++ b/cli/main.go @@ -41,6 +41,7 @@ func newRootCmd() *cobra.Command { cmd.NewGenerateCmd(), cmd.NewSetCmd(), cmd.NewVerifyCmd(), + cmd.NewRuntimeCmd(), ) return root diff --git a/packages/by-name/contrast/package.nix b/packages/by-name/contrast/package.nix index fa7bd0f60e..f4da360a32 100644 --- a/packages/by-name/contrast/package.nix +++ b/packages/by-name/contrast/package.nix @@ -66,6 +66,7 @@ buildGoModule rec { "-s" "-w" "-X main.version=v${version}" + "-X github.com/edgelesssys/contrast/cli/cmd.runtimeHandler=${runtimeHandler}" "-X github.com/edgelesssys/contrast/e2e/internal/kuberesource.runtimeHandler=${runtimeHandler}" ]; From ca0940a889d7e84257abc628dd25d1077801136d Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Wed, 17 Apr 2024 15:30:21 +0200 Subject: [PATCH 3/7] just: add runtime --- justfile | 18 +++++++++++++++++- packages/scripts.nix | 5 ++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/justfile b/justfile index dd2b3e24c4..3d2a975b5c 100644 --- a/justfile +++ b/justfile @@ -1,5 +1,5 @@ # Undeploy, rebuild, deploy. -default target=default_deploy_target cli=default_cli: soft-clean coordinator initializer openssl port-forwarder service-mesh-proxy (deploy target cli) set verify (wait-for-workload target) +default target=default_deploy_target cli=default_cli: soft-clean coordinator initializer openssl port-forwarder service-mesh-proxy node-installer runtime (apply "runtime") (deploy target cli) set verify (wait-for-workload target) # Build and push a container image. push target: @@ -31,6 +31,18 @@ workspace_dir := "workspace" # Generate policies, apply Kubernetes manifests. deploy target=default_deploy_target cli=default_cli: (populate target) (generate cli) (apply target) +# Populate the workspace with a runtime class deployment +runtime: + #!/usr/bin/env bash + set -euo pipefail + mkdir -p ./{{ workspace_dir }}/runtime + nix shell .#contrast --command resourcegen runtime ./{{ workspace_dir }}/runtime/runtime.yml + nix run .#scripts.patch-contrast-image-hashes -- ./{{ workspace_dir }}/runtime + nix run .#kypatch images -- ./{{ workspace_dir }}/runtime \ + --replace ghcr.io/edgelesssys ${container_registry} + nix run .#kypatch namespace -- ./{{ workspace_dir }}/runtime \ + --replace edg-default kube-system + # Populate the workspace with a Kubernetes deployment populate target=default_deploy_target: #!/usr/bin/env bash @@ -66,6 +78,10 @@ generate cli=default_cli: apply target=default_deploy_target: #!/usr/bin/env bash case {{ target }} in + "runtime") + kubectl apply -f ./{{ workspace_dir }}/runtime + exit 0 + ;; "simple" | "openssl" | "emojivoto") : ;; diff --git a/packages/scripts.nix b/packages/scripts.nix index 1bdf146271..e217369cc9 100644 --- a/packages/scripts.nix +++ b/packages/scripts.nix @@ -89,6 +89,7 @@ with pkgs; runtimeInputs = [ crane kypatch + jq ]; text = '' targetPath=$1 @@ -107,13 +108,15 @@ with pkgs; opensslHash=$(crane digest --tarball "$tmpdir/openssl.tar") forwarderHash=$(crane digest --tarball "$tmpdir/port-forwarder.tar") serviceMeshProxyHash=$(crane digest --tarball "$tmpdir/service-mesh-proxy.tar") + nodeInstallerHash=$(jq -r '.manifests[0].digest' "${contrast-node-installer-image}/index.json") kypatch images "$targetPath" \ --replace "contrast/coordinator:latest" "contrast/coordinator@$coordHash" \ --replace "contrast/initializer:latest" "contrast/initializer@$initHash" \ --replace "contrast/openssl:latest" "contrast/openssl@$opensslHash" \ --replace "contrast/port-forwarder:latest" "contrast/port-forwarder@$forwarderHash" \ - --replace "contrast/service-mesh-proxy:latest" "contrast/service-mesh-proxy@$serviceMeshProxyHash" + --replace "contrast/service-mesh-proxy:latest" "contrast/service-mesh-proxy@$serviceMeshProxyHash" \ + --replace "contrast/node-installer:latest" "contrast/node-installer@$nodeInstallerHash" ''; }; From 4e07189ba975e85ea54e6d3bfe64f4199baa3fcd Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Wed, 17 Apr 2024 13:41:50 +0200 Subject: [PATCH 4/7] deployments: use versioned contrast-cc runtime --- cli/cmd/generate.go | 8 ++++---- e2e/internal/kuberesource/parts.go | 2 +- e2e/internal/kuberesource/sets.go | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cli/cmd/generate.go b/cli/cmd/generate.go index 66c6236e3f..0ced6d302a 100644 --- a/cli/cmd/generate.go +++ b/cli/cmd/generate.go @@ -166,7 +166,7 @@ func findGenerateTargets(args []string, logger *slog.Logger) ([]string, error) { } } - paths = filterNonCoCoRuntime("kata-cc-isolation", paths, logger) + paths = filterNonCoCoRuntime("contrast-cc", paths, logger) if len(paths) == 0 { return nil, fmt.Errorf("no .yml/.yaml files found") @@ -174,7 +174,7 @@ func findGenerateTargets(args []string, logger *slog.Logger) ([]string, error) { return paths, nil } -func filterNonCoCoRuntime(runtimeClassName string, paths []string, logger *slog.Logger) []string { +func filterNonCoCoRuntime(runtimeClassNamePrefix string, paths []string, logger *slog.Logger) []string { var filtered []string for _, path := range paths { data, err := os.ReadFile(path) @@ -182,8 +182,8 @@ func filterNonCoCoRuntime(runtimeClassName string, paths []string, logger *slog. logger.Warn("Failed to read file", "path", path, "err", err) continue } - if !bytes.Contains(data, []byte(runtimeClassName)) { - logger.Info("Ignoring non-CoCo runtime", "className", runtimeClassName, "path", path) + if !bytes.Contains(data, []byte(runtimeClassNamePrefix)) { + logger.Info("Ignoring non-CoCo runtime", "className", runtimeClassNamePrefix, "path", path) continue } filtered = append(filtered, path) diff --git a/e2e/internal/kuberesource/parts.go b/e2e/internal/kuberesource/parts.go index 261b5fc160..2106117a83 100644 --- a/e2e/internal/kuberesource/parts.go +++ b/e2e/internal/kuberesource/parts.go @@ -141,7 +141,7 @@ func Coordinator(namespace string) *CoordinatorConfig { WithLabels(map[string]string{"app.kubernetes.io/name": "coordinator"}). WithAnnotations(map[string]string{"contrast.edgeless.systems/pod-role": "coordinator"}). WithSpec(PodSpec(). - WithRuntimeClassName("kata-cc-isolation"). + WithRuntimeClassName(runtimeHandler). WithContainers( Container(). WithName("coordinator"). diff --git a/e2e/internal/kuberesource/sets.go b/e2e/internal/kuberesource/sets.go index 78c0b5ac7b..357f251daa 100644 --- a/e2e/internal/kuberesource/sets.go +++ b/e2e/internal/kuberesource/sets.go @@ -60,7 +60,7 @@ func Simple() ([]any, error) { WithTemplate(PodTemplateSpec(). WithLabels(map[string]string{"app.kubernetes.io/name": "workload"}). WithSpec(PodSpec(). - WithRuntimeClassName("kata-cc-isolation"). + WithRuntimeClassName(runtimeHandler). WithContainers( Container(). WithName("workload"). @@ -105,7 +105,7 @@ func OpenSSL() ([]any, error) { WithTemplate(PodTemplateSpec(). WithLabels(map[string]string{"app.kubernetes.io/name": "openssl-backend"}). WithSpec(PodSpec(). - WithRuntimeClassName("kata-cc-isolation"). + WithRuntimeClassName(runtimeHandler). WithContainers( Container(). WithName("openssl-backend"). @@ -146,7 +146,7 @@ func OpenSSL() ([]any, error) { WithTemplate(PodTemplateSpec(). WithLabels(map[string]string{"app.kubernetes.io/name": "openssl-client"}). WithSpec(PodSpec(). - WithRuntimeClassName("kata-cc-isolation"). + WithRuntimeClassName(runtimeHandler). WithContainers( Container(). WithName("openssl-client"). @@ -173,7 +173,7 @@ func OpenSSL() ([]any, error) { WithTemplate(PodTemplateSpec(). WithLabels(map[string]string{"app.kubernetes.io/name": "openssl-frontend"}). WithSpec(PodSpec(). - WithRuntimeClassName("kata-cc-isolation"). + WithRuntimeClassName(runtimeHandler). WithContainers( Container(). WithName("openssl-frontend"). @@ -305,7 +305,7 @@ func generateEmojivoto(smMode serviceMeshMode) ([]any, error) { "version": "v11", }). WithSpec(PodSpec(). - WithRuntimeClassName("kata-cc-isolation"). + WithRuntimeClassName(runtimeHandler). WithServiceAccountName("emoji"). WithContainers( Container(). @@ -412,7 +412,7 @@ func generateEmojivoto(smMode serviceMeshMode) ([]any, error) { "version": "v11", }). WithSpec(PodSpec(). - WithRuntimeClassName("kata-cc-isolation"). + WithRuntimeClassName(runtimeHandler). WithServiceAccountName("voting"). WithContainers( Container(). @@ -485,7 +485,7 @@ func generateEmojivoto(smMode serviceMeshMode) ([]any, error) { "version": "v11", }). WithSpec(PodSpec(). - WithRuntimeClassName("kata-cc-isolation"). + WithRuntimeClassName(runtimeHandler). WithServiceAccountName("web"). WithContainers( Container(). From 3793b9aeaaa531be100f4c124ba130873b5e3e77 Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Wed, 17 Apr 2024 15:31:25 +0200 Subject: [PATCH 5/7] remove kata-cc-isolation from docs --- dev-docs/coco/policy.md | 2 +- docs/docs/deployment.md | 5 +++-- docs/docs/examples/emojivoto.md | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dev-docs/coco/policy.md b/dev-docs/coco/policy.md index 9e4b91b1c5..6439d9a1de 100644 --- a/dev-docs/coco/policy.md +++ b/dev-docs/coco/policy.md @@ -26,7 +26,7 @@ To trust the agent, we need to ensure that the agent only serves permitted reque For Contrast, the chain of trust looks like this: 1. The CLI generates a policy and attaches it to the pod definition. -2. Kubernetes schedules the pod on a node with `kata-cc-isolation` runtime. +2. Kubernetes schedules the pod on a node with a CoCo runtime. 3. Containerd takes the node, starts the Kata Shim and creates the pod sandbox. 4. The Kata runtime starts a CVM with the policy's digest as `HOSTDATA`. 5. The Kata runtime sets the policy using the `SetPolicy` method. diff --git a/docs/docs/deployment.md b/docs/docs/deployment.md index bddd239648..95fee72f72 100644 --- a/docs/docs/deployment.md +++ b/docs/docs/deployment.md @@ -47,13 +47,14 @@ cp -R $MY_RESOURCE_DIR resources/ To specify that a workload (pod, deployment, etc.) should be deployed as confidential containers, -add `runtimeClassName: kata-cc-isolation` to the pod spec (pod definition or template). +add `runtimeClassName: contrast-cc` to the pod spec (pod definition or template). +This is a placeholder name that will be replaced by a versioned `runtimeClassName` when generating policies. In addition, add the Contrast Initializer as `initContainers` to these workloads and configure the workload to use the certificates written to a `volumeMount` named `tls-certs`. ```yaml spec: # v1.PodSpec - runtimeClassName: kata-cc-isolation + runtimeClassName: contrast-cc initContainers: - name: initializer image: "ghcr.io/edgelesssys/contrast/initializer:latest" diff --git a/docs/docs/examples/emojivoto.md b/docs/docs/examples/emojivoto.md index 04444d1471..6e8429025e 100644 --- a/docs/docs/examples/emojivoto.md +++ b/docs/docs/examples/emojivoto.md @@ -57,7 +57,7 @@ contrast generate deployment/ :::note[Runtime class and Initializer] The deployment YAML shipped for this demo is already configured to be used with Contrast. -A runtime class `kata-cc-isolation` was added to the pods to signal they should be run +A runtime class `contrast-cc-` was added to the pods to signal they should be run as Confidential Containers. In addition, the Contrast Initializer was added as an init container to these workloads to facilitate the attestation and certificate pulling before the actual workload is started. From 9ad36724a93926c007d4ee3d8774ed53da27799f Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Wed, 17 Apr 2024 18:14:40 +0200 Subject: [PATCH 6/7] e2e: install runtime --- .github/workflows/e2e_openssl.yml | 2 +- .github/workflows/e2e_servicemesh.yml | 2 +- e2e/internal/contrasttest/contrasttest.go | 60 ++++++++++++++++----- e2e/internal/kubeclient/deploy.go | 66 ++++++++++++++++------- e2e/internal/kubeclient/kubeclient.go | 21 ++++++++ e2e/internal/kuberesource/sets.go | 13 +++++ e2e/openssl/openssl_test.go | 19 +------ e2e/servicemesh/servicemesh_test.go | 19 +------ packages/by-name/contrast/package.nix | 5 +- 9 files changed, 137 insertions(+), 70 deletions(-) diff --git a/.github/workflows/e2e_openssl.yml b/.github/workflows/e2e_openssl.yml index 23b8daeadb..0839550065 100644 --- a/.github/workflows/e2e_openssl.yml +++ b/.github/workflows/e2e_openssl.yml @@ -49,7 +49,7 @@ jobs: just get-credentials - name: Build and prepare deployments run: | - just coordinator initializer openssl port-forwarder + just coordinator initializer openssl port-forwarder node-installer - name: E2E Test run: | nix shell .#contrast.e2e --command openssl.test -test.v workspace/just.containerlookup diff --git a/.github/workflows/e2e_servicemesh.yml b/.github/workflows/e2e_servicemesh.yml index 9e21887271..b8aeebe7a9 100644 --- a/.github/workflows/e2e_servicemesh.yml +++ b/.github/workflows/e2e_servicemesh.yml @@ -49,7 +49,7 @@ jobs: just get-credentials - name: Build and prepare deployments run: | - just coordinator initializer port-forwarder service-mesh-proxy + just coordinator initializer port-forwarder service-mesh-proxy node-installer - name: E2E Test run: | nix shell .#contrast.e2e --command servicemesh.test -test.v workspace/just.containerlookup diff --git a/e2e/internal/contrasttest/contrasttest.go b/e2e/internal/contrasttest/contrasttest.go index da44ce5e86..553dff8f80 100644 --- a/e2e/internal/contrasttest/contrasttest.go +++ b/e2e/internal/contrasttest/contrasttest.go @@ -25,9 +25,10 @@ import ( // ContrastTest is the Contrast test helper struct. type ContrastTest struct { // inputs, usually filled by New() - Namespace string - WorkDir string - Kubeclient *kubeclient.Kubeclient + Namespace string + WorkDir string + ImageReplacements map[string]string + Kubeclient *kubeclient.Kubeclient // outputs of contrast subcommands coordinatorPolicyHash string @@ -36,16 +37,17 @@ type ContrastTest struct { } // New creates a new contrasttest.T object bound to the given test. -func New(t *testing.T) *ContrastTest { +func New(t *testing.T, imageReplacements map[string]string) *ContrastTest { return &ContrastTest{ - Namespace: makeNamespace(t), - WorkDir: t.TempDir(), - Kubeclient: kubeclient.NewForTest(t), + Namespace: makeNamespace(t), + WorkDir: t.TempDir(), + ImageReplacements: imageReplacements, + Kubeclient: kubeclient.NewForTest(t), } } // Init patches the given resources for the test environment and makes them available to Generate and Set. -func (ct *ContrastTest) Init(t *testing.T, objs []*unstructured.Unstructured) { +func (ct *ContrastTest) Init(t *testing.T, resources []any) { require := require.New(t) // Create namespace @@ -65,17 +67,26 @@ func (ct *ContrastTest) Init(t *testing.T, objs []*unstructured.Unstructured) { } }) - // Add the namespace for this test. - for _, obj := range objs { - require.NoError(ct.Kubeclient.PatchNamespace(ct.Namespace, obj)) + // Prepare resources + resources = kuberesource.PatchImages(resources, ct.ImageReplacements) + resources = kuberesource.PatchNamespaces(resources, ct.Namespace) + unstructuredResources, err := kuberesource.ResourcesToUnstructured(resources) + require.NoError(err) + var objects []*unstructured.Unstructured + for _, obj := range unstructuredResources { + // TODO(burgerdev): remove once demo deployments don't contain namespaces anymore. + if obj.GetKind() == "Namespace" { + continue + } + objects = append(objects, obj) } - // TODO(burgerdev): patch images - // Write resources to this test's tempdir. - buf, err := kuberesource.EncodeUnstructured(objs) + buf, err := kuberesource.EncodeUnstructured(objects) require.NoError(err) require.NoError(os.WriteFile(path.Join(ct.WorkDir, "resources.yaml"), buf, 0o644)) + + ct.installRuntime(t) } // Generate runs the contrast generate command. @@ -192,6 +203,27 @@ func (ct *ContrastTest) commonArgs() []string { } } +// installRuntime initializes the kubernetes runtime class for the test. +func (ct *ContrastTest) installRuntime(t *testing.T) { + require := require.New(t) + + resources, err := kuberesource.Runtime() + require.NoError(err) + + resources = kuberesource.PatchImages(resources, ct.ImageReplacements) + resources = kuberesource.PatchNamespaces(resources, ct.Namespace) + + unstructuredResources, err := kuberesource.ResourcesToUnstructured(resources) + require.NoError(err) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + require.NoError(ct.Kubeclient.Apply(ctx, unstructuredResources...)) + + require.NoError(ct.Kubeclient.WaitForDaemonset(ctx, ct.Namespace, "contrast-node-installer")) +} + func makeNamespace(t *testing.T) string { buf := make([]byte, 4) re := regexp.MustCompile("[a-z0-9-]+") diff --git a/e2e/internal/kubeclient/deploy.go b/e2e/internal/kubeclient/deploy.go index 69aa14e060..f37497ab6e 100644 --- a/e2e/internal/kubeclient/deploy.go +++ b/e2e/internal/kubeclient/deploy.go @@ -99,6 +99,53 @@ func (c *Kubeclient) WaitForDeployment(ctx context.Context, namespace, name stri } } +// WaitForDaemonset watches the given daemonset and blocks until the desired number of pods are +// ready or the context expires (is cancelled or times out). +func (c *Kubeclient) WaitForDaemonset(ctx context.Context, namespace, name string) error { + watcher, err := c.client.AppsV1().DaemonSets(namespace).Watch(ctx, metav1.ListOptions{FieldSelector: "metadata.name=" + name}) + if err != nil { + return err + } + for { + select { + case evt := <-watcher.ResultChan(): + switch evt.Type { + case watch.Added: + fallthrough + case watch.Modified: + ds, ok := evt.Object.(*appsv1.DaemonSet) + if !ok { + return fmt.Errorf("watcher received unexpected type %T", evt.Object) + } + if ds.Status.NumberReady >= ds.Status.DesiredNumberScheduled { + return nil + } + default: + return fmt.Errorf("unexpected watch event while waiting for daemonset %s/%s: %#v", namespace, name, evt.Object) + } + case <-ctx.Done(): + logger := c.log.With("namespace", namespace) + logger.Error("daemonset did not become ready", "name", name, "contextErr", ctx.Err()) + if ctx.Err() != context.DeadlineExceeded { + return ctx.Err() + } + // Fetch and print debug information. + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + pods, err := c.PodsFromDaemonSet(ctx, namespace, name) //nolint:contextcheck // The parent context expired. + if err != nil { + logger.Error("could not fetch pods for daemonset", "name", name, "error", err) + return ctx.Err() + } + for _, pod := range pods { + if !isPodReady(&pod) { + logger.Debug("pod not ready", "name", pod.Name, "status", c.toJSON(pod.Status)) + } + } + } + } +} + func (c *Kubeclient) toJSON(a any) string { s, err := json.Marshal(a) if err != nil { @@ -167,22 +214,3 @@ func (c *Kubeclient) Delete(ctx context.Context, objects ...*unstructured.Unstru } return nil } - -// PatchNamespace adjusts the namespace of the given object in-place if it is an instance of a namespaced resource. -func (c *Kubeclient) PatchNamespace(namespace string, obj *unstructured.Unstructured) error { - gvk := obj.GroupVersionKind() - resources, err := c.client.DiscoveryClient.ServerResourcesForGroupVersion(gvk.GroupVersion().String()) - if err != nil { - return fmt.Errorf("API resources not found for %#v: %w", gvk, err) - } - for _, resource := range resources.APIResources { - if resource.Kind != obj.GetKind() { - continue - } - if resource.Namespaced { - obj.SetNamespace(namespace) - } - return nil - } - return fmt.Errorf("API resource not found for %#v", gvk) -} diff --git a/e2e/internal/kubeclient/kubeclient.go b/e2e/internal/kubeclient/kubeclient.go index eaeef276af..4d9ed311d1 100644 --- a/e2e/internal/kubeclient/kubeclient.go +++ b/e2e/internal/kubeclient/kubeclient.go @@ -114,6 +114,27 @@ func (c *Kubeclient) PodsFromDeployment(ctx context.Context, namespace, deployme return out, nil } +// PodsFromDaemonSet returns the pods from a daemonset in a namespace. +// +// A pod is considered to belong to a daemonset if it is owned by the DaemonSet in question. +func (c *Kubeclient) PodsFromDaemonSet(ctx context.Context, namespace, daemonset string) ([]v1.Pod, error) { + pods, err := c.client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("listing pods: %w", err) + } + + var out []v1.Pod + for _, pod := range pods.Items { + for _, ref := range pod.OwnerReferences { + if ref.Kind == "DaemonSet" && ref.Name == daemonset { + out = append(out, pod) + } + } + } + + return out, nil +} + // Exec executes a process in a pod and returns the stdout and stderr. func (c *Kubeclient) Exec(ctx context.Context, namespace, pod string, argv []string) ( stdout string, stderr string, err error, diff --git a/e2e/internal/kuberesource/sets.go b/e2e/internal/kuberesource/sets.go index 357f251daa..23d39fec58 100644 --- a/e2e/internal/kuberesource/sets.go +++ b/e2e/internal/kuberesource/sets.go @@ -597,6 +597,17 @@ func PatchImages(resources []any, replacements map[string]string) []any { r.Spec.Template.Spec.Containers[i].Image = &replacement } } + case *applyappsv1.DaemonSetApplyConfiguration: + for i := 0; i < len(r.Spec.Template.Spec.InitContainers); i++ { + if replacement, ok := replacements[*r.Spec.Template.Spec.InitContainers[i].Image]; ok { + r.Spec.Template.Spec.InitContainers[i].Image = &replacement + } + } + for i := 0; i < len(r.Spec.Template.Spec.Containers); i++ { + if replacement, ok := replacements[*r.Spec.Template.Spec.Containers[i].Image]; ok { + r.Spec.Template.Spec.Containers[i].Image = &replacement + } + } case *applycorev1.PodApplyConfiguration: for i := 0; i < len(r.Spec.Containers); i++ { if replacement, ok := replacements[*r.Spec.Containers[i].Image]; ok { @@ -616,6 +627,8 @@ func PatchNamespaces(resources []any, namespace string) []any { r.Namespace = &namespace case *applyappsv1.DeploymentApplyConfiguration: r.Namespace = &namespace + case *applyappsv1.DaemonSetApplyConfiguration: + r.Namespace = &namespace case *applycorev1.ServiceApplyConfiguration: r.Namespace = &namespace case *applycorev1.ServiceAccountApplyConfiguration: diff --git a/e2e/openssl/openssl_test.go b/e2e/openssl/openssl_test.go index 38ef4dc3b7..7d9115ba35 100644 --- a/e2e/openssl/openssl_test.go +++ b/e2e/openssl/openssl_test.go @@ -17,7 +17,6 @@ import ( "github.com/edgelesssys/contrast/e2e/internal/kubeclient" "github.com/edgelesssys/contrast/e2e/internal/kuberesource" "github.com/stretchr/testify/require" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) // namespace the tests are executed in. @@ -30,26 +29,12 @@ var imageReplacements map[string]string // TestOpenSSL runs e2e tests on the example OpenSSL deployment. func TestOpenSSL(t *testing.T) { - ct := contrasttest.New(t) + ct := contrasttest.New(t, imageReplacements) resources, err := kuberesource.OpenSSL() require.NoError(t, err) - resources = kuberesource.PatchImages(resources, imageReplacements) - - unstructuredResources, err := kuberesource.ResourcesToUnstructured(resources) - require.NoError(t, err) - - var objects []*unstructured.Unstructured - for _, obj := range unstructuredResources { - // TODO(burgerdev): remove once demo deployments don't contain namespaces anymore. - if obj.GetKind() == "Namespace" { - continue - } - objects = append(objects, obj) - } - - ct.Init(t, objects) + ct.Init(t, resources) require.True(t, t.Run("generate", ct.Generate), "contrast generate needs to succeed for subsequent tests") require.True(t, t.Run("apply", ct.Apply), "Kubernetes resources need to be applied for subsequent tests") diff --git a/e2e/servicemesh/servicemesh_test.go b/e2e/servicemesh/servicemesh_test.go index cb8cadd0e5..00acf7f0d2 100644 --- a/e2e/servicemesh/servicemesh_test.go +++ b/e2e/servicemesh/servicemesh_test.go @@ -19,33 +19,18 @@ import ( "github.com/edgelesssys/contrast/e2e/internal/kubeclient" "github.com/edgelesssys/contrast/e2e/internal/kuberesource" "github.com/stretchr/testify/require" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) var imageReplacements map[string]string // TestIngressEgress tests that the ingress and egress proxies work as configured. func TestIngressEgress(t *testing.T) { - ct := contrasttest.New(t) + ct := contrasttest.New(t, imageReplacements) resources, err := kuberesource.EmojivotoIngressEgress() require.NoError(t, err) - resources = kuberesource.PatchImages(resources, imageReplacements) - - unstructuredResources, err := kuberesource.ResourcesToUnstructured(resources) - require.NoError(t, err) - - var objects []*unstructured.Unstructured - for _, obj := range unstructuredResources { - // TODO(burgerdev): remove once demo deployments don't contain namespaces anymore. - if obj.GetKind() == "Namespace" { - continue - } - objects = append(objects, obj) - } - - ct.Init(t, objects) + ct.Init(t, resources) require.True(t, t.Run("generate", ct.Generate), "contrast generate needs to succeed for subsequent tests") diff --git a/packages/by-name/contrast/package.nix b/packages/by-name/contrast/package.nix index f4da360a32..eb1bf9ca79 100644 --- a/packages/by-name/contrast/package.nix +++ b/packages/by-name/contrast/package.nix @@ -13,7 +13,10 @@ let tags = [ "e2e" ]; - ldflags = [ "-s" ]; + ldflags = [ + "-s" + "-X github.com/edgelesssys/contrast/e2e/internal/kuberesource.runtimeHandler=${runtimeHandler}" + ]; subPackages = [ "e2e/openssl" "e2e/servicemesh" ]; }; From 89538ed1ca796aaebf060285a03788b3c783f005 Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Thu, 18 Apr 2024 09:29:46 +0200 Subject: [PATCH 7/7] release: publish runtime class and installer as artifacts --- .github/workflows/release.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 87f76e07cd..bf2624ee7e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -187,17 +187,27 @@ jobs: - name: Push containers with release tag run: | coordinatorImg=$(nix run .#containers.push-coordinator -- "$container_registry/contrast/coordinator") + nodeInstallerImg=$(nix run .#containers.push-node-installer -- "$container_registry/contrast/node-installer") nix run .#containers.push-initializer -- "$container_registry/contrast/initializer" echo "coordinatorImg=$coordinatorImg" | tee -a "$GITHUB_ENV" + echo "nodeInstallerImg=$nodeInstallerImg" | tee -a "$GITHUB_ENV" - name: Add tag to Coordinator image run: | - front=${coordinatorImg%@*} - back=${coordinatorImg#*@} - echo "coordinatorImgTagged=${front}:${{ inputs.version }}@${back}" | tee -a "$GITHUB_ENV" + frontCoord=${coordinatorImg%@*} + backCoord=${coordinatorImg#*@} + echo "coordinatorImgTagged=${frontCoord}:${{ inputs.version }}@${backCoord}" | tee -a "$GITHUB_ENV" + frontNodeInstaller=${nodeInstallerImg%@*} + backNodeInstaller=${nodeInstallerImg#*@} + echo "nodeInstallerImgTagged=${frontNodeInstaller}:${{ inputs.version }}@${backNodeInstaller}" | tee -a "$GITHUB_ENV" - name: Create portable coordinator resource definitions run: | mkdir -p workspace nix run .#scripts.write-coordinator-yaml -- "${coordinatorImgTagged}" > workspace/coordinator.yml + nix shell .#contrast --command resourcegen runtime workspace/runtime.yml + nix run .#kypatch images -- workspace/runtime.yml \ + --replace "ghcr.io/edgelesssys/contrast/node-installer:latest" "$nodeInstallerImgTagged" + nix run .#kypatch namespace -- workspace/runtime.yml \ + --replace edg-default kube-system - name: Update coordinator policy hash run: | yq < workspace/coordinator.yml \ @@ -221,6 +231,7 @@ jobs: files: | result-cli/bin/contrast workspace/coordinator.yml + workspace/runtime.yml - name: Reset temporary changes run: | git reset --hard ${{ needs.process-inputs.outputs.WORKING_BRANCH }}