e2e: generate deployment yaml

e2e/internal/kuberesource/mutators.go

@@ -0,0 +1,49 @@
+package kuberesource
+
+import (
+	"errors"
+
+	applyappsv1 "k8s.io/client-go/applyconfigurations/apps/v1"
+	applycorev1 "k8s.io/client-go/applyconfigurations/core/v1"
+)
+
+// AddInitializer adds an initializer to a deployment.
+func AddInitializer(
+	deployment *applyappsv1.DeploymentApplyConfiguration,
+	initializer *applycorev1.ContainerApplyConfiguration,
+) (*applyappsv1.DeploymentApplyConfiguration, error) {
+	if initializer == nil {
+		return nil, errors.New("initializer is nil")
+	}
+	if deployment == nil {
+		return nil, errors.New("deployment is nil")
+	}
+	if deployment.Spec == nil {
+		return nil, errors.New("deployment.Spec is nil")
+	}
+	if deployment.Spec.Template == nil {
+		return nil, errors.New("deployment.Spec.Template is nil")
+	}
+	if deployment.Spec.Template.Spec == nil {
+		return nil, errors.New("deployment.Spec.Template.Spec is nil")
+	}
+	if len(deployment.Spec.Template.Spec.Containers) == 0 {
+		return nil, errors.New("deployment.Spec.Template.Spec.Containers is empty")
+	}
+
+	// Add the initializer as an init container.
+	deployment.Spec.Template.Spec.WithInitContainers(
+		initializer,
+	)
+	// Create the volume written by the initializer.
+	deployment.Spec.Template.Spec.WithVolumes(Volume().
+		WithName("tls-certs").
+		WithEmptyDir(EmptyDirVolumeSource().Inner()),
+	)
+	// Add the volume mount written by the initializer to the worker container.
+	deployment.Spec.Template.Spec.Containers[0].WithVolumeMounts(VolumeMount().
+		WithName("tls-certs").
+		WithMountPath("/tls-config"),
+	)
+	return deployment, nil
+}

e2e/internal/kuberesource/parts.go

@@ -0,0 +1,149 @@
+package kuberesource
+
+import (
+	"strconv"
+
+	applyappsv1 "k8s.io/client-go/applyconfigurations/apps/v1"
+	applycorev1 "k8s.io/client-go/applyconfigurations/core/v1"
+)
+
+// PortForwarderConfig wraps a PodApplyConfiguration for a port forwarder.
+type PortForwarderConfig struct {
+	*applycorev1.PodApplyConfiguration
+}
+
+// PortForwarder constructs a port forwarder pod.
+func PortForwarder(name, namespace string) *PortForwarderConfig {
+	name = "port-forwarder-" + name
+
+	p := Pod(name, namespace).
+		WithLabels(map[string]string{"app.kubernetes.io/name": name}).
+		WithSpec(PodSpec().
+			WithContainers(
+				Container().
+					WithName("port-forwarder").
+					WithImage("ghcr.io/edgelesssys/nunki/port-forwarder:latest").
+					WithCommand("/bin/bash", "-c", "echo Starting port-forward with socat; exec socat -d -d TCP-LISTEN:${LISTEN_PORT},fork TCP:${FORWARD_HOST}:${FORWARD_PORT}").
+					WithResources(ResourceRequirements().
+						WithMemoryLimitAndRequest(50),
+					),
+			),
+		)
+
+	return &PortForwarderConfig{p}
+}
+
+// WithListenPort sets the port to listen on.
+func (p *PortForwarderConfig) WithListenPort(port int32) *PortForwarderConfig {
+	p.Spec.Containers[0].
+		WithPorts(
+			ContainerPort().
+				WithContainerPort(port),
+		).
+		WithEnv(
+			NewEnvVar("LISTEN_PORT", strconv.Itoa(int(port))),
+		)
+	return p
+}
+
+// WithForwardTarget sets the target host and port to forward to.
+func (p *PortForwarderConfig) WithForwardTarget(host string, port int32) *PortForwarderConfig {
+	p.Spec.Containers[0].
+		WithEnv(
+			NewEnvVar("FORWARD_HOST", host),
+			NewEnvVar("FORWARD_PORT", strconv.Itoa(int(port))),
+		)
+	return p
+}
+
+// CoordinatorConfig wraps applyappsv1.DeploymentApplyConfiguration for a coordinator.
+type CoordinatorConfig struct {
+	*applyappsv1.DeploymentApplyConfiguration
+}
+
+// Coordinator constructs a new CoordinatorConfig.
+func Coordinator(namespace string) *CoordinatorConfig {
+	c := Deployment("coordinator", namespace).
+		WithSpec(DeploymentSpec().
+			WithReplicas(1).
+			WithSelector(LabelSelector().
+				WithMatchLabels(map[string]string{"app.kubernetes.io/name": "coordinator"}),
+			).
+			WithTemplate(PodTemplateSpec().
+				WithLabels(map[string]string{"app.kubernetes.io/name": "coordinator"}).
+				WithAnnotations(map[string]string{"nunki.edgeless.systems/pod-role": "coordinator"}).
+				WithSpec(PodSpec().
+					WithRuntimeClassName("kata-cc-isolation").
+					WithContainers(
+						Container().
+							WithName("coordinator").
+							WithImage("ghcr.io/edgelesssys/nunki/coordinator:latest").
+							WithEnv(
+								NewEnvVar("NUNKI_LOG_LEVEL", "debug"),
+							).
+							WithPorts(
+								ContainerPort().
+									WithName("userapi").
+									WithContainerPort(1313),
+								ContainerPort().
+									WithName("meshapi").
+									WithContainerPort(7777),
+							).
+							WithResources(ResourceRequirements().
+								WithMemoryLimitAndRequest(100),
+							),
+					),
+				),
+			),
+		)
+
+	return &CoordinatorConfig{c}
+}
+
+// WithImage sets the image of the coordinator.
+func (c *CoordinatorConfig) WithImage(image string) *CoordinatorConfig {
+	c.Spec.Template.Spec.Containers[0].WithImage(image)
+	return c
+}
+
+// GetDeploymentConfig returns the DeploymentConfig of the coordinator.
+func (c *CoordinatorConfig) GetDeploymentConfig() *DeploymentConfig {
+	return &DeploymentConfig{c.DeploymentApplyConfiguration}
+}
+
+// ServiceForDeployment creates a service for a deployment by exposing the configured ports
+// of the deployment's first container.
+func ServiceForDeployment(d *applyappsv1.DeploymentApplyConfiguration) *applycorev1.ServiceApplyConfiguration {
+	selector := d.Spec.Selector.MatchLabels
+	ports := d.Spec.Template.Spec.Containers[0].Ports
+
+	s := Service(*d.Name, *d.Namespace).
+		WithSpec(ServiceSpec().
+			WithSelector(selector),
+		)
+
+	for _, p := range ports {
+		s.Spec.WithPorts(
+			ServicePort().
+				WithName(*p.Name).
+				WithPort(*p.ContainerPort),
+		)
+	}
+
+	return s
+}
+
+// Initializer creates a new InitializerConfig.
+func Initializer() *applycorev1.ContainerApplyConfiguration {
+	return applycorev1.Container().
+		WithName("initializer").
+		WithImage("ghcr.io/edgelesssys/nunki/initializer:latest").
+		WithResources(ResourceRequirements().
+			WithMemoryLimitAndRequest(50),
+		).
+		WithEnv(NewEnvVar("COORDINATOR_HOST", "coordinator")).
+		WithVolumeMounts(VolumeMount().
+			WithName("tls-certs").
+			WithMountPath("/tls-config"),
+		)
+}

e2e/internal/kuberesource/parts_test.go

@@ -0,0 +1,29 @@
+package kuberesource
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewPortForwarder(t *testing.T) {
+	require := require.New(t)
+
+	config := PortForwarder("coordinator", "default").
+		WithListenPort(1313).
+		WithForwardTarget("coordinator", 1313)
+
+	b, err := EncodeResources(config)
+	require.NoError(err)
+	t.Log("\n" + string(b))
+}
+
+func TestCoordinator(t *testing.T) {
+	require := require.New(t)
+
+	config := Coordinator("default")
+
+	b, err := EncodeResources(config)
+	require.NoError(err)
+	t.Log("\n" + string(b))
+}

e2e/internal/kuberesource/resourcegen/main.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/edgelesssys/nunki/e2e/internal/kuberesource"
+)
+
+func main() {
+	if len(os.Args) != 3 {
+		fmt.Println("Usage: kuberesource <set> <dest>")
+		os.Exit(1)
+	}
+
+	set := os.Args[1]
+	dest := os.Args[2]
+
+	var resources []any
+	var err error
+	switch set {
+	case "simple":
+		resources, err = kuberesource.Simple()
+	default:
+		fmt.Printf("Error: unknown set: %s\n", set)
+		os.Exit(1)
+	}
+	if err != nil {
+		fmt.Printf("Error: %v\n", err)
+		os.Exit(1)
+	}
+
+	b, err := kuberesource.EncodeResources(resources...)
+	if err != nil {
+		fmt.Printf("Error: %v\n", err)
+		os.Exit(1)
+	}
+
+	if err := os.Mkdir(path.Dir(dest), 0o755); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	if err := os.WriteFile(dest, b, 0o644); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}

e2e/internal/kuberesource/sets.go

@@ -0,0 +1,51 @@
+package kuberesource
+
+// Simple returns a simple set of resources for testing.
+func Simple() ([]any, error) {
+	ns := "edg-default"
+
+	namespace := Namespace(ns)
+	coordinator := Coordinator(ns).DeploymentApplyConfiguration
+	coordinatorService := ServiceForDeployment(coordinator)
+	coordinatorForwarder := PortForwarder("coordinator", ns).
+		WithListenPort(1313).
+		WithForwardTarget("coordinator", 1313).
+		PodApplyConfiguration
+
+	workload := Deployment("workload", ns).
+		WithSpec(DeploymentSpec().
+			WithReplicas(1).
+			WithSelector(LabelSelector().
+				WithMatchLabels(map[string]string{"app.kubernetes.io/name": "workload"}),
+			).
+			WithTemplate(PodTemplateSpec().
+				WithLabels(map[string]string{"app.kubernetes.io/name": "workload"}).
+				WithSpec(PodSpec().
+					WithRuntimeClassName("kata-cc-isolation").
+					WithContainers(
+						Container().
+							WithName("workload").
+							WithImage("docker.io/library/busybox:1.36.1-musl@sha256:d4707523ce6e12afdbe9a3be5ad69027150a834870ca0933baf7516dd1fe0f56").
+							WithCommand("/bin/sh", "-c", "echo Workload started ; while true; do sleep 60; done").
+							WithResources(ResourceRequirements().
+								WithMemoryLimitAndRequest(50),
+							),
+					),
+				),
+			),
+		)
+	workload, err := AddInitializer(workload, Initializer())
+	if err != nil {
+		return nil, err
+	}
+
+	resources := []any{
+		namespace,
+		coordinator,
+		coordinatorService,
+		coordinatorForwarder,
+		workload,
+	}
+
+	return resources, nil
+}