From c6351d6b72758777d0ba8e1cd2ffab2ba2643adc Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Wed, 17 Apr 2024 11:14:08 +0200 Subject: [PATCH] node installer k8s deployment --- e2e/internal/kuberesource/constants.go | 4 ++ e2e/internal/kuberesource/parts.go | 67 +++++++++++++++++++ e2e/internal/kuberesource/resourcegen/main.go | 2 + e2e/internal/kuberesource/sets.go | 15 +++++ e2e/internal/kuberesource/wrappers.go | 67 +++++++++++++++++++ packages/by-name/contrast/package.nix | 4 ++ 6 files changed, 159 insertions(+) create mode 100644 e2e/internal/kuberesource/constants.go diff --git a/e2e/internal/kuberesource/constants.go b/e2e/internal/kuberesource/constants.go new file mode 100644 index 000000000..86eda695b --- /dev/null +++ b/e2e/internal/kuberesource/constants.go @@ -0,0 +1,4 @@ +package kuberesource + +// This value is injected at build time. +var runtimeHandler = "contrast-cc" diff --git a/e2e/internal/kuberesource/parts.go b/e2e/internal/kuberesource/parts.go index 7155bcbc2..bae81ca6f 100644 --- a/e2e/internal/kuberesource/parts.go +++ b/e2e/internal/kuberesource/parts.go @@ -3,11 +3,78 @@ package kuberesource import ( "strconv" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/intstr" applyappsv1 "k8s.io/client-go/applyconfigurations/apps/v1" applycorev1 "k8s.io/client-go/applyconfigurations/core/v1" ) +// ContrastRuntimeClass creates a new RuntimeClassConfig. +func ContrastRuntimeClass() *RuntimeClassConfig { + r := RuntimeClass(runtimeHandler). + WithHandler(runtimeHandler). + WithLabels(map[string]string{"addonmanager.kubernetes.io/mode": "Reconcile"}). + WithOverhead(Overhead(corev1.ResourceList{"memory": resource.MustParse("2Gi")})). + WithScheduling(Scheduling(map[string]string{"kubernetes.azure.com/kata-cc-isolation": "true"})) + + return &RuntimeClassConfig{r} +} + +// NodeInstallerConfig wraps a DaemonSetApplyConfiguration for a node installer. +type NodeInstallerConfig struct { + *applyappsv1.DaemonSetApplyConfiguration +} + +// NodeInstaller constructs a node installer daemon set. +func NodeInstaller(namespace string) *NodeInstallerConfig { + name := "contrast-node-installer" + + d := DaemonSet(name, namespace). + WithLabels(map[string]string{"app.kubernetes.io/name": name}). + WithSpec(DaemonSetSpec(). + WithSelector(LabelSelector(). + WithMatchLabels(map[string]string{"app.kubernetes.io/name": name}), + ). + WithTemplate(PodTemplateSpec(). + WithLabels(map[string]string{"app.kubernetes.io/name": name}). + WithAnnotations(map[string]string{"contrast.edgeless.systems/pod-role": "contrast-node-installer"}). + WithSpec(PodSpec(). + WithHostPID(true). + WithInitContainers(Container(). + WithName("installer"). + WithImage("ghcr.io/edgelesssys/contrast/node-installer:latest"). + WithResources(ResourceRequirements(). + WithMemoryLimitAndRequest(100), + ). + WithSecurityContext(PrivilegedSecurityContext()). + WithVolumeMounts(VolumeMount(). + WithName("host-mount"). + WithMountPath("/host")), + ). + WithContainers( + Container(). + WithName("pause"). + WithImage("k8s.gcr.io/pause"). + WithResources(ResourceRequirements(). + WithMemoryLimitAndRequest(10), + ), + ). + WithVolumes( + Volume(). + WithName("host-mount"). + WithHostPath(HostPathVolumeSource(). + WithPath("/"). + WithType(corev1.HostPathDirectory), + ), + ), + ), + ), + ) + + return &NodeInstallerConfig{d} +} + // PortForwarderConfig wraps a PodApplyConfiguration for a port forwarder. type PortForwarderConfig struct { *applycorev1.PodApplyConfiguration diff --git a/e2e/internal/kuberesource/resourcegen/main.go b/e2e/internal/kuberesource/resourcegen/main.go index b9ebcd7be..344dceb1a 100644 --- a/e2e/internal/kuberesource/resourcegen/main.go +++ b/e2e/internal/kuberesource/resourcegen/main.go @@ -22,6 +22,8 @@ func main() { switch set { case "coordinator-release": resources, err = kuberesource.CoordinatorRelease() + case "runtime": + resources, err = kuberesource.Runtime() case "simple": resources, err = kuberesource.Simple() case "openssl": diff --git a/e2e/internal/kuberesource/sets.go b/e2e/internal/kuberesource/sets.go index 7443922d4..59220e9b2 100644 --- a/e2e/internal/kuberesource/sets.go +++ b/e2e/internal/kuberesource/sets.go @@ -22,6 +22,21 @@ func CoordinatorRelease() ([]any, error) { return resources, nil } +// Runtime returns a set of resources for registering and installing the runtime. +func Runtime() ([]any, error) { + ns := "edg-default" + + runtimeClass := ContrastRuntimeClass().RuntimeClassApplyConfiguration + nodeInstaller := NodeInstaller(ns).DaemonSetApplyConfiguration + + resources := []any{ + runtimeClass, + nodeInstaller, + } + + return resources, nil +} + // Simple returns a simple set of resources for testing. func Simple() ([]any, error) { ns := "edg-default" diff --git a/e2e/internal/kuberesource/wrappers.go b/e2e/internal/kuberesource/wrappers.go index 939b75bd3..cb2430811 100644 --- a/e2e/internal/kuberesource/wrappers.go +++ b/e2e/internal/kuberesource/wrappers.go @@ -7,6 +7,7 @@ import ( applyappsv1 "k8s.io/client-go/applyconfigurations/apps/v1" applycorev1 "k8s.io/client-go/applyconfigurations/core/v1" applymetav1 "k8s.io/client-go/applyconfigurations/meta/v1" + applynodev1 "k8s.io/client-go/applyconfigurations/node/v1" ) // DeploymentConfig wraps applyappsv1.DeploymentApplyConfiguration. @@ -33,6 +34,30 @@ func DeploymentSpec() *DeploymentSpecConfig { return &DeploymentSpecConfig{applyappsv1.DeploymentSpec()} } +// DaemonSetConfig wraps applyappsv1.DaemonSetApplyConfiguration. +type DaemonSetConfig struct { + *applyappsv1.DaemonSetApplyConfiguration +} + +// DaemonSet creates a new DaemonSetConfig. +func DaemonSet(name, namespace string) *DaemonSetConfig { + d := applyappsv1.DaemonSet(name, namespace) + if namespace == "" && d.ObjectMetaApplyConfiguration != nil { + d.ObjectMetaApplyConfiguration.Namespace = nil + } + return &DaemonSetConfig{d} +} + +// DaemonSetSpecConfig wraps applyappsv1.DaemonSetSpecApplyConfiguration. +type DaemonSetSpecConfig struct { + *applyappsv1.DaemonSetSpecApplyConfiguration +} + +// DaemonSetSpec creates a new DaemonSetSpecConfig. +func DaemonSetSpec() *DaemonSetSpecConfig { + return &DaemonSetSpecConfig{applyappsv1.DaemonSetSpec()} +} + // PodConfig wraps applyappsv1.PodApplyConfiguration. type PodConfig struct { *applycorev1.PodApplyConfiguration @@ -170,6 +195,21 @@ func (e *EmptyDirVolumeSourceConfig) Inner() *applycorev1.EmptyDirVolumeSourceAp return e.EmptyDirVolumeSourceApplyConfiguration } +// HostPathVolumeSourceConfig wraps applycorev1.HostPathVolumeSourceApplyConfiguration. +type HostPathVolumeSourceConfig struct { + *applycorev1.HostPathVolumeSourceApplyConfiguration +} + +// HostPathVolumeSource creates a new HostPathVolumeSourceConfig. +func HostPathVolumeSource() *HostPathVolumeSourceConfig { + return &HostPathVolumeSourceConfig{applycorev1.HostPathVolumeSource()} +} + +// Inner returns the inner applycorev1.HostPathVolumeSourceApplyConfiguration. +func (h *HostPathVolumeSourceConfig) Inner() *applycorev1.HostPathVolumeSourceApplyConfiguration { + return h.HostPathVolumeSourceApplyConfiguration +} + // ContainerPortConfig wraps applycorev1.ContainerPortApplyConfiguration. type ContainerPortConfig struct { *applycorev1.ContainerPortApplyConfiguration @@ -180,6 +220,11 @@ func ContainerPort() *ContainerPortConfig { return &ContainerPortConfig{applycorev1.ContainerPort()} } +// PrivilegedSecurityContext returns a SecurityContextApplyConfiguration with Privileged set to true. +func PrivilegedSecurityContext() *applycorev1.SecurityContextApplyConfiguration { + return applycorev1.SecurityContext().WithPrivileged(true) +} + // ServiceConfig wraps applycorev1.ServiceApplyConfiguration. type ServiceConfig struct { *applycorev1.ServiceApplyConfiguration @@ -244,6 +289,28 @@ func TCPSocketAction() *applycorev1.TCPSocketActionApplyConfiguration { return applycorev1.TCPSocketAction() } +// RuntimeClassConfig wraps applypodsv1.RuntimeClassApplyConfiguration for a runtime class. +type RuntimeClassConfig struct { + *applynodev1.RuntimeClassApplyConfiguration +} + +// RuntimeClass constructs a new RuntimeClassConfig. +func RuntimeClass(name string) *RuntimeClassConfig { + return &RuntimeClassConfig{applynodev1.RuntimeClass(name)} +} + +// Overhead creates a new OverheadApplyConfiguration. +func Overhead(podFixed corev1.ResourceList) *applynodev1.OverheadApplyConfiguration { + return applynodev1.Overhead().WithPodFixed(podFixed) +} + +// Scheduling creates a new SchedulingApplyConfiguration. +func Scheduling(nodeSelector map[string]string, tolerations ...*applycorev1.TolerationApplyConfiguration) *applynodev1.SchedulingApplyConfiguration { + return applynodev1.Scheduling(). + WithNodeSelector(nodeSelector). + WithTolerations(tolerations...) +} + func fromPtr[T any](v *T) T { if v != nil { return *v diff --git a/packages/by-name/contrast/package.nix b/packages/by-name/contrast/package.nix index 91281fdd2..fa7bd0f60 100644 --- a/packages/by-name/contrast/package.nix +++ b/packages/by-name/contrast/package.nix @@ -4,6 +4,7 @@ , genpolicy-msft , genpolicy ? genpolicy-msft , contrast +, runtime-class-files }: let e2e = buildGoTest rec { @@ -17,6 +18,8 @@ let subPackages = [ "e2e/openssl" "e2e/servicemesh" ]; }; + runtimeHandler = lib.removeSuffix "\n" (builtins.readFile "${runtime-class-files}/runtime-handler"); + packageOutputs = [ "coordinator" "initializer" "cli" ]; in @@ -63,6 +66,7 @@ buildGoModule rec { "-s" "-w" "-X main.version=v${version}" + "-X github.com/edgelesssys/contrast/e2e/internal/kuberesource.runtimeHandler=${runtimeHandler}" ]; preCheck = ''