e2e: smoke test for peerpods

Co-authored-by: jmxnzo <[email protected]>

.github/workflows/e2e_peerpods.yml

@@ -0,0 +1,46 @@
+name: e2e peer-pods
+
+on:
+  workflow_dispatch:
+    inputs:
+      image-id:
+        description: "ID of the guest VM image to test (default: build a fresh image)"
+        required: false
+  pull_request:
+    paths:
+      - .github/workflows/e2e_peerpods.yml
+      - packages/test-peerpods.sh
+      - packages/by-name/cloud-api-adaptor/**
+      - packages/by-name/kata/**
+      - packages/by-name/image-podvm/**
+
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: ./.github/actions/setup_nix
+        with:
+          githubToken: ${{ secrets.GITHUB_TOKEN }}
+          cachixToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
+      - name: Login to Azure
+        uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0
+        with:
+          creds: ${{ secrets.CONTRAST_CI_INFRA_AZURE }}
+      - name: Test peer-pods
+        env:
+          azure_subscription_id: ${{ vars.AZURE_SUBSCRIPTION_ID }}
+          azure_tenant_id: ${{ vars.AZURE_TENANT_ID }}
+          azure_client_id: ${{ vars.PEER_POD_CLIENT_ID_AZURE }}
+          azure_client_secret: ${{ secrets.PEER_POD_CLIENT_SECRET_AZURE }}
+          azure_image_id: ${{ inputs.image-id }}
+          azure_resource_group: contrast-ci
+          azure_location: germanywestcentral
+          CONTRAST_CACHE_DIR: "./workspace.cache"
+        run: |
+          ssh-keygen -t rsa -f ./infra/azure-peerpods/id_rsa -N ""
+          nix run .#scripts.test-peerpods
+      - name: Terminate cluster
+        if: always()
+        run: |
+          nix run -L .#terraform -- -chdir=infra/azure-peerpods destroy --auto-approve

.gitignore

@@ -22,5 +22,4 @@ id_rsa*
 kube.conf
 out.env
 infra/**/kustomization.yaml
-infra/**/workload-identity.yaml
 uplosi.conf*

infra/azure-peerpods/main.tf

@@ -4,10 +4,6 @@ terraform {
       source  = "hashicorp/azurerm"
       version = "4.5.0"
     }
-    azuread = {
-      source  = "hashicorp/azuread"
-      version = "3.0.2"
-    }
     local = {
       source  = "hashicorp/local"
       version = "2.5.2"
@@ -26,59 +22,12 @@ provider "azurerm" {
 
 data "azurerm_subscription" "current" {}
 
-data "azuread_client_config" "current" {}
-
-provider "azuread" {
-  tenant_id = data.azurerm_subscription.current.tenant_id
-}
-
 locals {
   name = "${var.name_prefix}_caa_cluster"
 }
 
 data "azurerm_resource_group" "rg" {
-  name = local.name
-}
-
-resource "azuread_application" "app" {
-  display_name = local.name
-  owners       = [data.azuread_client_config.current.object_id]
-}
-
-resource "azuread_service_principal" "sp" {
-  client_id                    = azuread_application.app.client_id
-  app_role_assignment_required = false
-  owners                       = [data.azuread_client_config.current.object_id]
-}
-
-resource "azurerm_role_assignment" "ra_vm_contributor" {
-  scope                = data.azurerm_resource_group.rg.id
-  role_definition_name = "Virtual Machine Contributor"
-  principal_id         = azuread_service_principal.sp.object_id
-}
-
-resource "azurerm_role_assignment" "ra_reader" {
-  scope                = data.azurerm_resource_group.rg.id
-  role_definition_name = "Reader"
-  principal_id         = azuread_service_principal.sp.object_id
-}
-
-resource "azurerm_role_assignment" "ra_network_contributor" {
-  scope                = data.azurerm_resource_group.rg.id
-  role_definition_name = "Network Contributor"
-  principal_id         = azuread_service_principal.sp.object_id
-}
-
-resource "azuread_application_federated_identity_credential" "federated_credentials" {
-  display_name   = local.name
-  application_id = azuread_application.app.id
-  issuer         = azurerm_kubernetes_cluster.cluster.oidc_issuer_url
-  subject        = "system:serviceaccount:confidential-containers-system:cloud-api-adaptor"
-  audiences      = ["api://AzureADTokenExchange"]
-}
-
-resource "azuread_application_password" "cred" {
-  application_id = azuread_application.app.id
+  name = "${var.resource_group}"
 }
 
 resource "azurerm_virtual_network" "main" {
@@ -128,31 +77,6 @@ resource "local_file" "kubeconfig" {
   content    = azurerm_kubernetes_cluster.cluster.kube_config_raw
 }
 
-resource "local_file" "workload_identity" {
-  filename        = "./workload-identity.yaml"
-  file_permission = "0777"
-  content         = <<EOF
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: cloud-api-adaptor-daemonset
-  namespace: confidential-containers-system
-spec:
-  template:
-    metadata:
-      labels:
-        azure.workload.identity/use: "true"
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: cloud-api-adaptor
-  namespace: confidential-containers-system
-  annotations:
-    azure.workload.identity/client-id: ${azuread_application.app.client_id}
-EOF
-}
-
 resource "local_file" "kustomization" {
   filename        = "./kustomization.yaml"
   file_permission = "0777"
@@ -178,6 +102,9 @@ configMapGenerator:
   - AZURE_RESOURCE_GROUP=${data.azurerm_resource_group.rg.name}
   - AZURE_SUBNET_ID=${one(azurerm_virtual_network.main.subnet.*.id)}
   - AZURE_IMAGE_ID=${var.image_id}
+  - AZURE_CLIENT_ID=${var.client_id}
+  - AZURE_TENANT_ID=${var.tenant_id}
+  - AZURE_CLIENT_SECRET=${var.client_secret}
   - DISABLECVM=false
 secretGenerator:
 - name: peer-pods-secret
@@ -186,7 +113,5 @@ secretGenerator:
   namespace: confidential-containers-system
   files:
   - id_rsa.pub
-patchesStrategicMerge:
-- workload-identity.yaml
 EOF
 }

infra/azure-peerpods/vars.tf

@@ -2,14 +2,26 @@ variable "name_prefix" {
   type = string
 }
 
-variable "image_resource_group_name" {
+variable "resource_group" {
   type = string
 }
 
 variable "subscription_id" {
   type = string
 }
 
+variable "client_id" {
+  type = string
+}
+
+variable "tenant_id" {
+  type = string
+}
+
+variable "client_secret" {
+  type = string
+}
+
 variable "image_id" {
   type = string
 }

justfile

@@ -54,7 +54,6 @@ node-installer platform=default_platform:
         "AKS-PEER-SNP")
             nix run -L .#scripts.deploy-caa -- \
                 --kustomization=./infra/azure-peerpods/kustomization.yaml \
-                --workload-identity=./infra/azure-peerpods/workload-identity.yaml \
                 --pub-key=./infra/azure-peerpods/id_rsa.pub
         ;;
         *)
@@ -201,7 +200,7 @@ create platform=default_platform:
             echo "subscription_id = \"$azure_subscription_id\"" >> infra/azure-peerpods/just.auto.tfvars
 
             nix run -L .#terraform -- -chdir=infra/azure-peerpods init
-            nix run -L .#terraform -- -chdir=infra/azure-peerpods apply
+            nix run -L .#terraform -- -chdir=infra/azure-peerpods apply --auto-approve
         ;;
         *)
             echo "Unsupported platform: {{ platform }}"
@@ -330,7 +329,7 @@ destroy platform=default_platform:
             :
         ;;
         "AKS-PEER-SNP")
-            nix run -L .#terraform -- -chdir=infra/azure-peerpods destroy
+            nix run -L .#terraform -- -chdir=infra/azure-peerpods destroy --auto-approve
 
             # Clean-up cached image ids.
             rm -f ${CONTRAST_CACHE_DIR}/image-upload/*.image-id

packages/scripts.nix

@@ -465,10 +465,6 @@
           kustomizationFile="''${i#*=}"
           shift
           ;;
-        --workload-identity=*)
-          workloadIdentityFile="''${i#*=}"
-          shift
-          ;;
         --pub-key=*)
           pubKeyFile="''${i#*=}"
           shift
@@ -484,7 +480,6 @@
       cp -r ${pkgs.cloud-api-adaptor.src}/src/cloud-api-adaptor/install/* "$tmpdir"
       chmod -R +w "$tmpdir"
       cp "$kustomizationFile" "$tmpdir/overlays/azure/kustomization.yaml"
-      cp "$workloadIdentityFile" "$tmpdir/overlays/azure/workload-identity.yaml"
       cp "$pubKeyFile" "$tmpdir/overlays/azure/id_rsa.pub"
 
       kubectl apply -k "github.com/confidential-containers/operator/config/release?ref=v${pkgs.cloud-api-adaptor.version}"
@@ -513,4 +508,14 @@
     ];
     text = builtins.readFile ./cleanup-images.sh;
   };
+
+  test-peerpods = writeShellApplication {
+    name = "test-peerpods";
+    runtimeInputs = with pkgs; [
+      just
+      azure-cli
+      kubectl
+    ];
+    text = builtins.readFile ./test-peerpods.sh;
+  };
 }

packages/test-peerpods.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Copyright 2024 Edgeless Systems GmbH
+# SPDX-License-Identifier: AGPL-3.0-only
+
+set -euo pipefail
+
+set -x
+
+if [ -z "${azure_image_id}" ]; then
+  nix run -L .#scripts.upload-image -- \
+    --subscription-id="${azure_subscription_id:?}" \
+    --location="${azure_location:?}" \
+    --resource-group="${azure_resource_group:?}"
+else
+  echo "image_id = \"${azure_image_id}\"" > infra/azure-peerpods/image_id.auto.tfvars
+fi
+
+
+cat >infra/azure-peerpods/testrun.auto.tfvars <<EOF
+name_prefix = "${azure_resource_group:?}-$RANDOM"
+resource_group = "${azure_resource_group:?}"
+subscription_id = "${azure_subscription_id:?}"
+tenant_id = "${azure_tenant_id:?}"
+client_id = "${azure_client_id:?}"
+client_secret = "${azure_client_secret:?}"
+EOF
+
+nix run -L .#terraform -- -chdir=infra/azure-peerpods init
+nix run -L .#terraform -- -chdir=infra/azure-peerpods apply --auto-approve
+
+just get-credentials AKS-PEER-SNP
+just node-installer AKS-PEER-SNP
+
+cleanup() {
+  kubectl delete deploy nginx
+  kubectl wait --for=delete pod --selector=app=nginx --timeout=5m
+}
+
+trap cleanup EXIT
+
+kubectl apply -f - <<EOF
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+spec:
+  selector:
+    matchLabels:
+      app: nginx
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      runtimeClassName: kata-remote
+      containers:
+      - name: nginx
+        image: nginx
+        imagePullPolicy: Always
+EOF
+
+if ! kubectl wait --for=condition=available --timeout=5m deployment/nginx; then
+    kubectl describe pods
+    exit 1
+fi