e2e: add test helper struct for contrast subcommands

.github/workflows/e2e_openssl.yml

@@ -36,18 +36,11 @@ jobs:
         with:
           creds: ${{ secrets.CONTRAST_CI_INFRA_AZURE }}
       - uses: nicknovitski/nix-develop@a2060d116a50b36dfab02280af558e73ab52427d # v1.1.0
-      - name: Generate namespace suffix
-        id: ns
-        run: |
-          uuid=$(cat /proc/sys/kernel/random/uuid)
-          uid=${uuid##*-}
-          echo "namespace_suffix=$uid" >> "$GITHUB_OUTPUT"
       - name: Create justfile.env
         run: |
           cat <<EOF > justfile.env
           container_registry=${{ env.container_registry }}
           azure_resource_group=${{ env.azure_resource_group }}
-          namespace_suffix=-${{ steps.ns.outputs.namespace_suffix }}
           EOF
       - name: Get credentials for CI cluster
         run: |
@@ -58,12 +51,7 @@ jobs:
           just populate openssl
       - name: Setup Summary
         run: |
-          cat ./workspace/just.namespace | tee -a "${GITHUB_STEP_SUMMARY}"
           cat ./workspace/just.perf | tee -a "${GITHUB_STEP_SUMMARY}"
       - name: E2E Test
         run: |
-          env K8S_NAMESPACE=$(cat ./workspace/just.namespace) nix shell .#contrast.e2e --command openssl.test -test.v
-      - name: Undeploy
-        if: always() && inputs.skip-undeploy != 'true'
-        run: |
-          just undeploy
+          nix shell .#contrast.e2e --command openssl.test -test.v

e2e/internal/contrasttest/contrasttest.go

@@ -0,0 +1,203 @@
+package contrasttest
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/x509"
+	"encoding/hex"
+	"io"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/edgelesssys/contrast/cli/cmd"
+	"github.com/edgelesssys/contrast/e2e/internal/kubeclient"
+	"github.com/edgelesssys/contrast/e2e/internal/kuberesource"
+	"github.com/edgelesssys/contrast/internal/kubeapi"
+	"github.com/stretchr/testify/require"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+)
+
+// ContrastTest is the Contrast test helper struct.
+type ContrastTest struct {
+	// inputs, usually filled by New()
+	Namespace  string
+	WorkDir    string
+	Kubeclient *kubeclient.Kubeclient
+
+	// outputs of contrast subcommands
+	coordinatorPolicyHash string
+	meshCACertPEM         []byte
+	rootCACertPEM         []byte
+}
+
+// New creates a new contrasttest.T object bound to the given test.
+func New(t *testing.T) *ContrastTest {
+	return &ContrastTest{
+		Namespace:  makeNamespace(t),
+		WorkDir:    t.TempDir(),
+		Kubeclient: kubeclient.NewForTest(t),
+	}
+}
+
+// Init patches the given resources for the test environment and makes them available to Generate and Set.
+func (ct *ContrastTest) Init(t *testing.T, objs []*unstructured.Unstructured) {
+	require := require.New(t)
+
+	// Create namespace
+	namespace, err := kuberesource.ResourcesToUnstructured([]any{kuberesource.Namespace(ct.Namespace)})
+	require.NoError(err)
+	// Creating a namespace should not take too long.
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	err = ct.Kubeclient.Apply(ctx, namespace...)
+	cancel()
+	require.NoError(err)
+	t.Cleanup(func() {
+		// Deleting the namespace may take some time due to pod cleanup, but we don't want to wait until the test times out.
+		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+		defer cancel()
+		if err := ct.Kubeclient.Delete(ctx, namespace...); err != nil {
+			t.Logf("Could not delete namespace %q: %v", ct.Namespace, err)
+		}
+	})
+
+	// Add the namespace for this test.
+	for _, obj := range objs {
+		require.NoError(ct.Kubeclient.PatchNamespace(ct.Namespace, obj))
+	}
+
+	// TODO(burgerdev): patch images
+
+	// Write resources to this test's tempdir.
+	buf, err := kuberesource.EncodeUnstructured(objs)
+	require.NoError(err)
+	require.NoError(os.WriteFile(path.Join(ct.WorkDir, "resources.yaml"), buf, 0o644))
+}
+
+// Generate runs the contrast generate command.
+func (ct *ContrastTest) Generate(t *testing.T) {
+	require := require.New(t)
+
+	args := append(ct.commonArgs(), path.Join(ct.WorkDir, "resources.yaml"))
+
+	generate := cmd.NewGenerateCmd()
+	generate.Flags().String("workspace-dir", "", "") // Make generate aware of root flags
+	generate.SetArgs(args)
+	generate.SetOut(io.Discard)
+	errBuf := &bytes.Buffer{}
+	generate.SetErr(errBuf)
+
+	require.NoError(generate.Execute(), "could not generate manifest: %s", errBuf)
+	hash, err := os.ReadFile(path.Join(ct.WorkDir, "coordinator-policy.sha256"))
+	require.NoError(err)
+	require.NotEmpty(hash, "expected apply to fill coordinator policy hash")
+	ct.coordinatorPolicyHash = string(hash)
+}
+
+// Apply the generated resources to the Kubernetes test environment.
+func (ct *ContrastTest) Apply(t *testing.T) {
+	require := require.New(t)
+
+	yaml, err := os.ReadFile(path.Join(ct.WorkDir, "resources.yaml"))
+	require.NoError(err)
+	objects, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml)
+	require.NoError(err)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+	defer cancel()
+
+	require.NoError(ct.Kubeclient.Apply(ctx, objects...))
+}
+
+// Set runs the contrast set subcommand.
+func (ct *ContrastTest) Set(t *testing.T) {
+	require := require.New(t)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+	defer cancel()
+
+	require.NoError(ct.Kubeclient.WaitForDeployment(ctx, ct.Namespace, "coordinator"))
+
+	coordinator, cancelPortForward, err := ct.Kubeclient.PortForwardPod(ctx, ct.Namespace, "port-forwarder-coordinator", "1313")
+	require.NoError(err)
+	defer cancelPortForward()
+
+	args := append(ct.commonArgs(),
+		"--coordinator-policy-hash", ct.coordinatorPolicyHash,
+		"--coordinator", coordinator,
+		path.Join(ct.WorkDir, "resources.yaml"))
+
+	set := cmd.NewSetCmd()
+	set.Flags().String("workspace-dir", "", "") // Make set aware of root flags
+	set.SetArgs(args)
+	set.SetOut(io.Discard)
+	errBuf := &bytes.Buffer{}
+	set.SetErr(errBuf)
+
+	require.NoError(set.Execute(), "could not set manifest at coordinator: %s", errBuf)
+}
+
+// Verify runs the contrast verify subcommand.
+func (ct *ContrastTest) Verify(t *testing.T) {
+	require := require.New(t)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+	defer cancel()
+
+	require.NoError(ct.Kubeclient.WaitForDeployment(ctx, ct.Namespace, "coordinator"))
+
+	coordinator, cancelPortForward, err := ct.Kubeclient.PortForwardPod(ctx, ct.Namespace, "port-forwarder-coordinator", "1313")
+	require.NoError(err)
+	defer cancelPortForward()
+
+	verify := cmd.NewVerifyCmd()
+	verify.SetArgs(append(
+		ct.commonArgs(),
+		"--coordinator-policy-hash", ct.coordinatorPolicyHash,
+		"--coordinator", coordinator,
+	))
+	verify.SetOut(io.Discard)
+	errBuf := &bytes.Buffer{}
+	verify.SetErr(errBuf)
+
+	require.NoError(verify.Execute(), "could not verify coordinator: %s", errBuf)
+
+	ct.meshCACertPEM, err = os.ReadFile(path.Join(ct.WorkDir, "mesh-root.pem"))
+	require.NoError(err)
+	ct.rootCACertPEM, err = os.ReadFile(path.Join(ct.WorkDir, "coordinator-root.pem"))
+	require.NoError(err)
+}
+
+// MeshCACert returns a CertPool that contains the coordinator mesh CA cert.
+func (ct *ContrastTest) MeshCACert() *x509.CertPool {
+	pool := x509.NewCertPool()
+	pool.AppendCertsFromPEM(ct.meshCACertPEM)
+	return pool
+}
+
+// RootCACert returns a CertPool that contains the coordinator root CA cert.
+func (ct *ContrastTest) RootCACert() *x509.CertPool {
+	pool := x509.NewCertPool()
+	pool.AppendCertsFromPEM(ct.rootCACertPEM)
+	return pool
+}
+
+func (ct *ContrastTest) commonArgs() []string {
+	return []string{
+		"--workspace-dir", ct.WorkDir,
+	}
+}
+
+func makeNamespace(t *testing.T) string {
+	buf := make([]byte, 4)
+	re := regexp.MustCompile("[a-z0-9-]+")
+	n, err := rand.Reader.Read(buf)
+	require.NoError(t, err)
+	require.Equal(t, 4, n)
+
+	return strings.Join(append(re.FindAllString(strings.ToLower(t.Name()), -1), hex.EncodeToString(buf)), "-")
+}

e2e/internal/kubeclient/deploy.go

@@ -112,3 +112,38 @@ func (c *Kubeclient) Apply(ctx context.Context, objects ...*unstructured.Unstruc
 	}
 	return nil
 }
+
+// Delete a set of manifests.
+func (c *Kubeclient) Delete(ctx context.Context, objects ...*unstructured.Unstructured) error {
+	for _, obj := range objects {
+		ri, err := c.resourceInterfaceFor(obj)
+		if err != nil {
+			return err
+		}
+
+		if err := ri.Delete(ctx, obj.GetName(), metav1.DeleteOptions{}); err != nil {
+			return fmt.Errorf("could not apply %s %s in namespace %s: %w", obj.GetKind(), obj.GetName(), obj.GetNamespace(), err)
+		}
+		c.log.Info("object deleted", "namespace", obj.GetNamespace(), "kind", obj.GetKind(), "name", obj.GetName())
+	}
+	return nil
+}
+
+// PatchNamespace adjusts the namespace of the given object in-place if it is an instance of a namespaced resource.
+func (c *Kubeclient) PatchNamespace(namespace string, obj *unstructured.Unstructured) error {
+	gvk := obj.GroupVersionKind()
+	resources, err := c.client.DiscoveryClient.ServerResourcesForGroupVersion(gvk.GroupVersion().String())
+	if err != nil {
+		return fmt.Errorf("API resources not found for %#v: %w", gvk, err)
+	}
+	for _, resource := range resources.APIResources {
+		if resource.Kind != obj.GetKind() {
+			continue
+		}
+		if resource.Namespaced {
+			obj.SetNamespace(namespace)
+		}
+		return nil
+	}
+	return fmt.Errorf("API resource not found for %#v", gvk)
+}