e2e: add volumestatefulset test

.github/workflows/e2e.yaml

@@ -35,7 +35,7 @@ jobs:
           - name: K3s-QEMU-TDX
             runner: TDX
             self-hosted: true
-        test_name: [servicemesh, openssl, policy, workloadsecret]
+        test_name: [servicemesh, openssl, policy, workloadsecret, volumestatefulset]
         exclude:
           # We don't have policies on K3s-qemu yet, so there's no point in
           # running those tests.
@@ -49,6 +49,17 @@ jobs:
               runner: TDX
               self-hosted: true
             test_name: policy
+          # We don't have the policy support for CSI storage on K3s-qemu yet
+          - platform:
+              name: K3s-QEMU-SNP
+              runner: SNP
+              self-hosted: true
+            test_name: volumestatefulset
+          - platform:
+              name: K3s-QEMU-TDX
+              runner: TDX
+              self-hosted: true
+            test_name: volumestatefulset
       fail-fast: false
     name: "${{ matrix.platform.name }} / ${{ matrix.test_name }}"
     runs-on: ${{ matrix.platform.runner }}
@@ -93,7 +104,7 @@ jobs:
           echo "SYNC_FIFO_UUID=$sync_uuid" | tee -a "$GITHUB_ENV"
       - name: Build and prepare deployments
         run: |
-          just coordinator initializer port-forwarder openssl service-mesh-proxy node-installer ${{ matrix.platform.name }}
+          just coordinator initializer port-forwarder openssl cryptsetup service-mesh-proxy node-installer ${{ matrix.platform.name }}
       - name: E2E Test
         run: |
           nix run .#scripts.get-logs workspace/e2e.namespace &

e2e/volumestatefulset/volumestatefulset_test.go

@@ -0,0 +1,116 @@
+// Copyright 2024 Edgeless Systems GmbH
+// SPDX-License-Identifier: AGPL-3.0-only
+
+//go:build e2e
+
+package volumestatefulset
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/edgelesssys/contrast/e2e/internal/contrasttest"
+	"github.com/edgelesssys/contrast/e2e/internal/kubeclient"
+	"github.com/edgelesssys/contrast/internal/kuberesource"
+	"github.com/edgelesssys/contrast/internal/manifest"
+	"github.com/edgelesssys/contrast/internal/platforms"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	imageReplacementsFile, namespaceFile, platformStr string
+	skipUndeploy                                      bool
+)
+
+// TestWorkloadSecrets tests that secrets are correctly injected into workloads.
+func TestVolumeStatefulSet(t *testing.T) {
+	platform, err := platforms.FromString(platformStr)
+	require.NoError(t, err)
+	ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, platform, skipUndeploy)
+
+	runtimeHandler, err := manifest.RuntimeHandler(platform)
+	require.NoError(t, err)
+
+	resources := kuberesource.VolumeStatefulSet()
+
+	coordinator := kuberesource.CoordinatorBundle()
+
+	resources = append(resources, coordinator...)
+
+	resources = kuberesource.PatchRuntimeHandlers(resources, runtimeHandler)
+
+	resources = kuberesource.AddPortForwarders(resources)
+
+	ct.Init(t, resources)
+
+	require.True(t, t.Run("generate", ct.Generate), "contrast generate needs to succeed for subsequent tests")
+
+	require.True(t, t.Run("apply", ct.Apply), "Kubernetes resources need to be applied for subsequent tests")
+
+	require.True(t, t.Run("set", ct.Set), "contrast set needs to succeed for subsequent tests")
+
+	require.True(t, t.Run("contrast verify", ct.Verify), "contrast verify needs to succeed for subsequent tests")
+
+	require.True(t, t.Run("deployments become available", func(t *testing.T) {
+		require := require.New(t)
+
+		ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+		defer cancel()
+
+		require.NoError(ct.Kubeclient.WaitFor(ctx, kubeclient.StatefulSet{}, ct.Namespace, "volume-tester"))
+	}), "deployments need to be ready for subsequent tests")
+
+	filePath := "/srv/state/test"
+	t.Run("can create file in mounted path", func(t *testing.T) {
+		require := require.New(t)
+
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		pods, err := ct.Kubeclient.PodsFromOwner(ctx, ct.Namespace, "StatefulSet", "volume-tester")
+		require.NoError(err)
+		require.Len(pods, 1)
+
+		stdOut, stdErr, err := ct.Kubeclient.Exec(ctx, ct.Namespace, pods[0].Name, []string{"sh", "-c", fmt.Sprintf("echo test > %s", filePath)})
+		require.NoError(err, "stdout: %s, stderr: %s", stdOut, stdErr)
+
+		stdOut, stdErr, err = ct.Kubeclient.Exec(ctx, ct.Namespace, pods[0].Name, []string{"sync"})
+		require.NoError(err, "stdout: %s, stderr: %s", stdOut, stdErr)
+
+		stdOut, stdErr, err = ct.Kubeclient.Exec(ctx, ct.Namespace, pods[0].Name, []string{"cat", filePath})
+		require.NoError(err, "stdout: %s, stderr: %s", stdOut, stdErr)
+		require.Equal("test\n", stdOut)
+	})
+
+	t.Run("file still exists when pod is restarted", func(t *testing.T) {
+		require := require.New(t)
+
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		require.NoError(ct.Kubeclient.Restart(ctx, kubeclient.StatefulSet{}, ct.Namespace, "volume-tester"))
+		require.NoError(ct.Kubeclient.WaitFor(ctx, kubeclient.StatefulSet{}, ct.Namespace, "volume-tester"))
+
+		pods, err := ct.Kubeclient.PodsFromOwner(ctx, ct.Namespace, "StatefulSet", "volume-tester")
+		require.NoError(err)
+		require.Len(pods, 1)
+
+		stdOut, stdErr, err := ct.Kubeclient.Exec(ctx, ct.Namespace, pods[0].Name, []string{"cat", filePath})
+		require.NoError(err, "stdout: %s, stderr: %s", stdOut, stdErr)
+		require.Equal("test\n", stdOut)
+	})
+}
+
+func TestMain(m *testing.M) {
+	flag.StringVar(&imageReplacementsFile, "image-replacements", "", "path to image replacements file")
+	flag.StringVar(&namespaceFile, "namespace-file", "", "file to store the namespace in")
+	flag.StringVar(&platformStr, "platform", "", "Deployment platform")
+	flag.BoolVar(&skipUndeploy, "skip-undeploy", true, "skip undeploy step in the test")
+	flag.Parse()
+
+	os.Exit(m.Run())
+}

justfile

@@ -57,7 +57,7 @@ node-installer platform=default_platform:
         ;;
     esac
 
-e2e target=default_deploy_target platform=default_platform: soft-clean coordinator initializer openssl port-forwarder service-mesh-proxy (node-installer platform)
+e2e target=default_deploy_target platform=default_platform: soft-clean coordinator initializer cryptsetup openssl port-forwarder service-mesh-proxy (node-installer platform)
     #!/usr/bin/env bash
     set -euo pipefail
     nix shell .#contrast.e2e --command {{ target }}.test -test.v \

packages/by-name/contrast/package.nix

@@ -36,6 +36,7 @@ let
       "e2e/release"
       "e2e/policy"
       "e2e/workloadsecret"
+      "e2e/volumestatefulset"
       "e2e/regression"
     ];
   };