e2e: add a test for checking AKS upstream (#939)

.github/workflows/e2e_aks_runtime.yml

@@ -0,0 +1,107 @@
+name: e2e test aks runtime
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "16 6 * * 6" # 6:16 on Saturdays
+  pull_request:
+    paths:
+      - e2e/aks-runtime/**
+
+env:
+  container_registry: ghcr.io/edgelesssys
+  azure_resource_group: contrast-ci
+  DO_NOT_TRACK: 1
+
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: ./.github/actions/setup_nix
+        with:
+          githubToken: ${{ secrets.GITHUB_TOKEN }}
+          cachixToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
+      - name: Login to Azure
+        uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0
+        with:
+          creds: ${{ secrets.CONTRAST_CI_INFRA_AZURE }}
+      - name: Log in to ghcr.io Container registry
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Create justfile.env
+        run: |
+          cat <<EOF > justfile.env
+          container_registry=${{ env.container_registry }}
+          azure_resource_group=${{ env.azure_resource_group }}
+          EOF
+      - name: Get credentials for CI cluster
+        run: |
+          nix run .#just -- get-credentials
+      - name: Set sync environment
+        run: |
+          sync_ip=$(kubectl get svc sync -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+          echo "SYNC_ENDPOINT=http://$sync_ip:8080" | tee -a "$GITHUB_ENV"
+          sync_uuid=$(kubectl get configmap sync-server-fifo -o jsonpath='{.data.uuid}')
+          echo "SYNC_FIFO_UUID=$sync_uuid" | tee -a "$GITHUB_ENV"
+      - name: Build and prepare deployments
+        run: |
+          nix shell .#just --command just coordinator initializer port-forwarder openssl cryptsetup service-mesh-proxy node-installer AKS-CLH-SNP
+      # steps taken from https://learn.microsoft.com/en-us/cli/azure/install-azure-cli-linux?pivots=apt#option-2-step-by-step-installation-instructions
+      - name: Install `az` with extensions
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install apt-transport-https ca-certificates curl gnupg lsb-release
+          sudo mkdir -p /etc/apt/keyrings
+          curl -sLS https://packages.microsoft.com/keys/microsoft.asc |
+          gpg --dearmor | sudo tee /etc/apt/keyrings/microsoft.gpg > /dev/null
+          sudo chmod go+r /etc/apt/keyrings/microsoft.gpg
+          AZ_DIST=$(lsb_release -cs)
+
+          sudo tee /etc/apt/sources.list.d/azure-cli.sources <<EOF
+          Types: deb
+          URIs: https://packages.microsoft.com/repos/azure-cli/
+          Suites: ${AZ_DIST}
+          Components: main
+          Architectures: $(dpkg --print-architecture)
+          Signed-by: /etc/apt/keyrings/microsoft.gpg
+          EOF
+
+          sudo apt-get update
+          sudo apt-get -y install azure-cli
+
+          az extension add --name aks-preview
+          az extension add --name confcom
+      - name: E2E test
+        run: |
+          nix run .#scripts.get-logs workspace/e2e.namespace &
+          nix build .#contrast.e2e
+          ./result/bin/aks-runtime.test -test.v \
+            --image-replacements workspace/just.containerlookup \
+            --namespace-file workspace/e2e.namespace \
+            --platform AKS-CLH-SNP \
+            --skip-undeploy="false"
+      - name: Upload logs
+        if: always()
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        with:
+          name: e2e_pod_logs-AKS-CLH-SNP-aks-runtime
+          path: workspace/namespace-logs
+      - name: Notify teams channel of failure
+        if: ${{ failure() && github.event_name == 'schedule' && github.run_attempt == 1 }}
+        uses: ./.github/actions/post_to_teams
+        with:
+          webhook: ${{ secrets.TEAMS_CI_WEBHOOK }}
+          title: "aks-runtime test failed"
+          message: "e2e test aks-runtime failed"
+          additionalFields: '[{"title": "Platform", "value": "AKS-CLH-SNP"}]'
+      - name: Cleanup
+        if: cancelled()
+        run: |
+          kubectl delete ns "$(cat workspace/e2e.namespace)" --timeout 5m

e2e/aks-runtime/aks_runtime_test.go

@@ -0,0 +1,144 @@
+// Copyright 2024 Edgeless Systems GmbH
+// SPDX-License-Identifier: AGPL-3.0-only
+
+//go:build e2e
+
+package aksruntime
+
+import (
+	"context"
+	"flag"
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/edgelesssys/contrast/e2e/internal/az"
+	"github.com/edgelesssys/contrast/e2e/internal/contrasttest"
+	"github.com/edgelesssys/contrast/e2e/internal/kubeclient"
+	"github.com/edgelesssys/contrast/internal/kubeapi"
+	"github.com/edgelesssys/contrast/internal/kuberesource"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const testContainer = "testcontainer"
+
+var (
+	imageReplacementsFile, namespaceFile, _platformStr string
+	skipUndeploy                                       bool
+)
+
+func TestAKSRuntime(t *testing.T) {
+	require := require.New(t)
+
+	workdir := t.TempDir()
+
+	f, err := os.Open(imageReplacementsFile)
+	require.NoError(err)
+	imageReplacements, err := kuberesource.ImageReplacementsFromFile(f)
+	require.NoError(err)
+	namespace := contrasttest.MakeNamespace(t)
+
+	// Log versions
+	kataPolicyGenV, err := az.KataPolicyGenVersion()
+	require.NoError(err)
+	rg := os.Getenv("azure_resource_group")
+	nodeImageV, err := az.NodeImageVersion(rg, rg)
+	require.NoError(err)
+	t.Log("katapolicygen version: ", kataPolicyGenV)
+	t.Log("node image version: ", nodeImageV)
+
+	c := kubeclient.NewForTest(t)
+
+	// create the namespace
+	ns, err := kuberesource.ResourcesToUnstructured([]any{kuberesource.Namespace(namespace)})
+	require.NoError(err)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	err = c.Apply(ctx, ns...)
+	cancel()
+	require.NoError(err)
+	if namespaceFile != "" {
+		require.NoError(os.WriteFile(namespaceFile, []byte(namespace), 0o644))
+	}
+
+	// simple deployment that logs the kernel version and then sleeps
+	deployment := kuberesource.Deployment(testContainer, "").
+		WithSpec(kuberesource.DeploymentSpec().
+			WithReplicas(1).
+			WithSelector(kuberesource.LabelSelector().WithMatchLabels(
+				map[string]string{"app.kubernetes.io/name": testContainer},
+			)).
+			WithTemplate(kuberesource.PodTemplateSpec().
+				WithLabels(map[string]string{"app.kubernetes.io/name": testContainer}).
+				WithSpec(kuberesource.PodSpec().
+					WithContainers(kuberesource.Container().
+						WithName(testContainer).
+						WithImage("ghcr.io/edgelesssys/bash@sha256:cabc70d68e38584052cff2c271748a0506b47069ebbd3d26096478524e9b270b").
+						WithCommand("/usr/local/bin/bash", "-c", "uname -r; sleep infinity"),
+					),
+				),
+			),
+		)
+
+	// define resources
+	resources := []any{deployment}
+	resources = kuberesource.PatchRuntimeHandlers(resources, "kata-cc-isolation")
+	resources = kuberesource.PatchNamespaces(resources, namespace)
+	resources = kuberesource.PatchImages(resources, imageReplacements)
+
+	toWrite, err := kuberesource.ResourcesToUnstructured(resources)
+	require.NoError(err)
+
+	// generate policies
+	resourceBytes, err := kuberesource.EncodeUnstructured(toWrite)
+	require.NoError(err)
+	require.NoError(os.WriteFile(path.Join(workdir, "resources.yaml"), resourceBytes, 0o644))
+	require.NoError(az.KataPolicyGen(path.Join(workdir, "resources.yaml")))
+
+	// load in generated resources
+	resourceBytes, err = os.ReadFile(path.Join(workdir, "resources.yaml"))
+	require.NoError(err)
+	toApply, err := kubeapi.UnmarshalUnstructuredK8SResource(resourceBytes)
+	require.NoError(err)
+
+	ctx, cancel = context.WithTimeout(context.Background(), 3*time.Minute)
+	defer cancel()
+	err = c.Apply(ctx, toApply...)
+	require.NoError(err)
+	require.NoError(c.WaitFor(ctx, kubeclient.Ready, kubeclient.Deployment{}, namespace, testContainer))
+
+	t.Cleanup(func() {
+		if skipUndeploy {
+			return
+		}
+
+		// delete the deployment
+		deletePolicy := metav1.DeletePropagationForeground
+		if err = c.Client.CoreV1().Namespaces().Delete(context.Background(), namespace, metav1.DeleteOptions{
+			PropagationPolicy: &deletePolicy,
+		}); err != nil {
+			t.Fatalf("Failed to delete namespace %s", namespace)
+		}
+	})
+
+	pods, err := c.Client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
+	require.NoError(err)
+	require.Len(pods.Items, 1)
+	pod := pods.Items[0] // only one pod was deployed
+
+	logs, err := c.Client.CoreV1().Pods(namespace).GetLogs(pod.Name, &corev1.PodLogOptions{}).DoRaw(ctx)
+	require.NoError(err)
+	t.Logf("kernel version in pod %s: %s", pod.Name, string(logs))
+}
+
+func TestMain(m *testing.M) {
+	flag.StringVar(&imageReplacementsFile, "image-replacements", "", "path to image replacements file")
+	flag.StringVar(&namespaceFile, "namespace-file", "", "file to store the namespace in")
+	flag.StringVar(&_platformStr, "platform", "", "Deployment platform")
+	flag.BoolVar(&skipUndeploy, "skip-undeploy", false, "skip undeploy step in the test")
+	flag.Parse()
+
+	os.Exit(m.Run())
+}

e2e/internal/az/aks.go

@@ -0,0 +1,34 @@
+// Copyright 2024 Edgeless Systems GmbH
+// SPDX-License-Identifier: AGPL-3.0-only
+
+//go:build e2e
+
+package az
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os/exec"
+	"strings"
+)
+
+// NodeImageVersion gets the node image version from the specified cluster
+// and resource group.
+func NodeImageVersion(clusterName string, rg string) (string, error) {
+	out, err := exec.Command("az", "aks", "nodepool", "list", "--cluster-name", clusterName, "--resource-group", rg).Output()
+	if err != nil {
+		return "", err
+	}
+
+	var outMap []map[string]interface{}
+	err = json.Unmarshal(out, &outMap)
+	if err != nil {
+		return "", err
+	}
+	if len(outMap) == 0 {
+		return "", errors.New("No nodepools could be listed")
+	}
+
+	return strings.TrimSpace(fmt.Sprintf("%s", outMap[0]["nodeImageVersion"])), nil
+}

e2e/internal/az/confcom.go

@@ -0,0 +1,28 @@
+// Copyright 2024 Edgeless Systems GmbH
+// SPDX-License-Identifier: AGPL-3.0-only
+
+//go:build e2e
+
+package az
+
+import (
+	"os"
+	"os/exec"
+	"strings"
+)
+
+// KataPolicyGenVersion gets the version string of `az confcom katapolicygen`.
+func KataPolicyGenVersion() (string, error) {
+	out, err := exec.Command("az", "confcom", "katapolicygen", "--print-version").Output()
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(string(out)), nil
+}
+
+// KataPolicyGen executes `az confcom katapolicygen --yaml <resourcePath>`.
+func KataPolicyGen(resourcePath string) error {
+	cmd := exec.Command("az", "confcom", "katapolicygen", "--yaml", resourcePath)
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}

e2e/internal/contrasttest/contrasttest.go

@@ -52,7 +52,7 @@ type ContrastTest struct {
 // New creates a new contrasttest.T object bound to the given test.
 func New(t *testing.T, imageReplacements, namespaceFile string, platform platforms.Platform, skipUndeploy bool) *ContrastTest {
 	return &ContrastTest{
-		Namespace:             makeNamespace(t),
+		Namespace:             MakeNamespace(t),
 		WorkDir:               t.TempDir(),
 		ImageReplacementsFile: imageReplacements,
 		Platform:              platform,
@@ -372,7 +372,8 @@ func (ct *ContrastTest) FactorPlatformTimeout(timeout time.Duration) time.Durati
 	}
 }
 
-func makeNamespace(t *testing.T) string {
+// MakeNamespace creates a namespace string using a given *testing.T.
+func MakeNamespace(t *testing.T) string {
 	buf := make([]byte, 4)
 	re := regexp.MustCompile("[a-z0-9-]+")
 	n, err := rand.Reader.Read(buf)

flake.nix

@@ -31,6 +31,10 @@
           overlays = [ (import ./overlays/nixpkgs.nix) ];
           config.allowUnfree = true;
           config.nvidia.acceptLicense = true;
+          # TODO(miampf): REMOVE AGAIN ONCE UNNEEDED
+          config.permittedInsecurePackages = [
+            "openssl-1.1.1w"
+          ];
         };
         inherit (pkgs) lib;
         treefmtEval = treefmt-nix.lib.evalModule pkgs ./treefmt.nix;