From f6ee5bb27404500a8a53bb4ddc1519b2d93f9118 Mon Sep 17 00:00:00 2001
From: Markus Rudy <mr@edgeless.systems>
Date: Fri, 15 Nov 2024 15:55:22 +0100
Subject: [PATCH] e2e: smoke test for peer pods

Co-authored-by: Paul Meyer <49727155+katexochen@users.noreply.github.com>
---
 .github/workflows/e2e_peerpods.yml | 49 +++++++++++++++++++++++
 packages/scripts.nix               | 10 +++++
 packages/test-peerpods.sh          | 62 ++++++++++++++++++++++++++++++
 3 files changed, 121 insertions(+)
 create mode 100644 .github/workflows/e2e_peerpods.yml
 create mode 100644 packages/test-peerpods.sh

diff --git a/.github/workflows/e2e_peerpods.yml b/.github/workflows/e2e_peerpods.yml
new file mode 100644
index 000000000..bfac9aa6b
--- /dev/null
+++ b/.github/workflows/e2e_peerpods.yml
@@ -0,0 +1,49 @@
+name: e2e peer-pods
+
+on:
+  workflow_dispatch:
+    inputs:
+      image-id:
+        description: "ID of the guest VM image to test (default: build a fresh image)"
+        required: false
+  pull_request:
+    paths:
+      - .github/workflows/e2e_peerpods.yml
+      - packages/test-peerpods.sh
+      - packages/by-name/cloud-api-adaptor/**
+      - packages/by-name/kata/**
+      - packages/by-name/image-podvm/**
+      - packages/nixos
+
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: ./.github/actions/setup_nix
+        with:
+          githubToken: ${{ secrets.GITHUB_TOKEN }}
+          cachixToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
+      - name: Login to Azure
+        uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0
+        with:
+          creds: ${{ secrets.CONTRAST_CI_INFRA_AZURE }}
+      - name: Test peer-pods
+        env:
+          azure_subscription_id: ${{ vars.AZURE_SUBSCRIPTION_ID }}
+          azure_image_id: ${{ inputs.image-id }}
+          azure_resource_group: contrast-ci
+          azure_location: germanywestcentral
+          CONTRAST_CACHE_DIR: "./workspace.cache"
+        run: |
+          ssh-keygen -t rsa -f ./infra/azure-peerpods/id_rsa -N ""
+          cat >infra/azure-peerpods/iam.auto.tfvars <<EOF
+          tenant_id = "${{ vars.AZURE_TENANT_ID }}"
+          client_id = "${{ vars.PEER_POD_CLIENT_ID_AZURE }}"
+          client_secret = "${{ secrets.PEER_POD_CLIENT_SECRET_AZURE }}"
+          EOF
+          nix run .#scripts.test-peerpods
+      - name: Terminate cluster
+        if: always()
+        run: |
+          nix run -L .#terraform -- -chdir=infra/azure-peerpods destroy --auto-approve
diff --git a/packages/scripts.nix b/packages/scripts.nix
index 22fabeee4..908556ab3 100644
--- a/packages/scripts.nix
+++ b/packages/scripts.nix
@@ -509,4 +509,14 @@
     ];
     text = builtins.readFile ./cleanup-images.sh;
   };
+
+  test-peerpods = writeShellApplication {
+    name = "test-peerpods";
+    runtimeInputs = with pkgs; [
+      just
+      azure-cli
+      kubectl
+    ];
+    text = builtins.readFile ./test-peerpods.sh;
+  };
 }
diff --git a/packages/test-peerpods.sh b/packages/test-peerpods.sh
new file mode 100644
index 000000000..b8fc53f2f
--- /dev/null
+++ b/packages/test-peerpods.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# Copyright 2024 Edgeless Systems GmbH
+# SPDX-License-Identifier: AGPL-3.0-only
+
+set -euo pipefail
+
+set -x
+
+if [ -z "${azure_image_id}" ]; then
+  nix run -L .#scripts.upload-image -- \
+    --subscription-id="${azure_subscription_id:?}" \
+    --location="${azure_location:?}" \
+    --resource-group="${azure_resource_group:?}"
+else
+  echo "image_id = \"${azure_image_id}\"" >infra/azure-peerpods/image_id.auto.tfvars
+fi
+
+cat >infra/azure-peerpods/just.auto.tfvars <<EOF
+name_prefix = "${azure_resource_group:?}-$RANDOM"
+resource_group = "${azure_resource_group:?}"
+subscription_id = "${azure_subscription_id:?}"
+EOF
+
+nix run -L .#terraform -- -chdir=infra/azure-peerpods init
+nix run -L .#terraform -- -chdir=infra/azure-peerpods apply --auto-approve
+
+just get-credentials AKS-PEER-SNP
+just node-installer AKS-PEER-SNP
+
+cleanup() {
+  kubectl delete deploy nginx
+  kubectl wait --for=delete pod --selector=app=nginx --timeout=5m
+}
+
+trap cleanup EXIT
+
+kubectl apply -f - <<EOF
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+spec:
+  selector:
+    matchLabels:
+      app: nginx
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      runtimeClassName: kata-remote
+      containers:
+      - name: nginx
+        image: nginx
+        imagePullPolicy: Always
+EOF
+
+if ! kubectl wait --for=condition=available --timeout=5m deployment/nginx; then
+  kubectl describe pods
+  exit 1
+fi