From f735a9afc79b3f97fea9251b4e9746197c0b90dc Mon Sep 17 00:00:00 2001
From: Markus Rudy <mr@edgeless.systems>
Date: Wed, 13 Nov 2024 14:20:42 +0100
Subject: [PATCH 1/4] scripts: create ~/.kube if it does not exist

---
 packages/scripts.nix | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/scripts.nix b/packages/scripts.nix
index 02d762ad9..9050e1fe1 100644
--- a/packages/scripts.nix
+++ b/packages/scripts.nix
@@ -403,7 +403,9 @@
       newContext=$(yq -r '.contexts.[0].name' "$1")
       declare -x newContext
       yq -i '.current-context = env(newContext)' "$mergedConfig"
-      mv "$mergedConfig" "''${KUBECONFIG_BAK%%:*}"
+      targetFile="''${KUBECONFIG_BAK%%:*}"
+      mkdir -p "$(dirname "$targetFile")"
+      mv "$mergedConfig" "$targetFile"
     '';
   };
 

From a84a9cbf75c61b03e045ed34c637c677d40e8169 Mon Sep 17 00:00:00 2001
From: Markus Rudy <mr@edgeless.systems>
Date: Fri, 15 Nov 2024 08:55:03 +0100
Subject: [PATCH 2/4] scripts: support dashes in resource group for
 upload-image

---
 packages/scripts.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/scripts.nix b/packages/scripts.nix
index 9050e1fe1..92d9934ee 100644
--- a/packages/scripts.nix
+++ b/packages/scripts.nix
@@ -69,7 +69,7 @@
         subscriptionID = "''${subscriptionId}"
         location = "''${location}"
         resourceGroup = "''${resourceGroup}"
-        sharedImageGallery = "''${resourceGroup}_contrast"
+        sharedImageGallery = "''${resourceGroup//-/_}_contrast"
         sharingProfile = "private"
         EOF
 

From d48ca25af94c3089778f93f5ed6225f2c4d5ad42 Mon Sep 17 00:00:00 2001
From: Markus Rudy <mr@edgeless.systems>
Date: Fri, 15 Nov 2024 15:54:38 +0100
Subject: [PATCH 3/4] infra: separate IAM from AKS and image

Co-authored-by: Paul Meyer <49727155+katexochen@users.noreply.github.com>
---
 .gitignore                                   |  1 -
 infra/azure-peerpods-iam/.terraform.lock.hcl | 62 ++++++++++++++
 infra/azure-peerpods-iam/main.tf             | 75 +++++++++++++++++
 infra/azure-peerpods-iam/outs.tf             |  8 ++
 infra/azure-peerpods-iam/vars.tf             | 11 +++
 infra/azure-peerpods/.terraform.lock.hcl     | 30 -------
 infra/azure-peerpods/main.tf                 | 89 ++------------------
 infra/azure-peerpods/vars.tf                 | 17 +++-
 justfile                                     | 69 ++++++++++++---
 packages/scripts.nix                         |  5 --
 10 files changed, 235 insertions(+), 132 deletions(-)
 create mode 100644 infra/azure-peerpods-iam/.terraform.lock.hcl
 create mode 100644 infra/azure-peerpods-iam/main.tf
 create mode 100644 infra/azure-peerpods-iam/outs.tf
 create mode 100644 infra/azure-peerpods-iam/vars.tf

diff --git a/.gitignore b/.gitignore
index 3f764fc51..9e0eb180e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,5 +22,4 @@ id_rsa*
 kube.conf
 out.env
 infra/**/kustomization.yaml
-infra/**/workload-identity.yaml
 uplosi.conf*
diff --git a/infra/azure-peerpods-iam/.terraform.lock.hcl b/infra/azure-peerpods-iam/.terraform.lock.hcl
new file mode 100644
index 000000000..cf2b56c5a
--- /dev/null
+++ b/infra/azure-peerpods-iam/.terraform.lock.hcl
@@ -0,0 +1,62 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/azuread" {
+  version     = "3.0.2"
+  constraints = "3.0.2"
+  hashes = [
+    "h1:sYCyzbPpSYu2XDah8XqBUITQAfB0x4j4Twh6lw2C4CA=",
+    "zh:16e724b80a9004c7978c30f69a73c98ff63eb8a03937dd44c2a8f0ea0438b7a3",
+    "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7",
+    "zh:2bbbf13713ca4767267b889471c9fc14a56a8fdf5d1013da3ca78667e3caec64",
+    "zh:409ccb05431d643a079da082d89db2d95d6afed4769997ac537c8b7de3bff867",
+    "zh:53e4bca0f5d015380f7f524f36344afe6211ccaf614bfc69af73ca64a9f47d6c",
+    "zh:5780be2c1981d090604d7fa4cef675462f17f40e7f3dc501a031488e87a35b8f",
+    "zh:850e61a1b3e64c752c418526ccf48653514c861b36f5feb631619f906f7e99a0",
+    "zh:8c3565bfcea006a734149cc080452a9daf7d2a9d5362eb7e0a088b6c0d7f0f03",
+    "zh:908b9e6ad49d5d21173ecefc7924902047611be93bbf8e7d021aa9563358396f",
+    "zh:a2a79765c029bc58966eff61cb6e9b0ee14d2ac52b0a22fc7dfa35c9a49af669",
+    "zh:c7f56cbe8743e9ba81fce871bc97d9c07abe86770d9ee7ffefbf3882a61ba89a",
+    "zh:d4dba80e33421b30d81c62611fb7fc62ad39afecc6484436e635913cd8553e67",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/azurerm" {
+  version     = "4.5.0"
+  constraints = "4.5.0"
+  hashes = [
+    "h1:bAEb9HTc1Yl0ULs+WQAI6jAoKWv4I2LUGpoESf/iCyc=",
+    "zh:27ac12977bdb7b82217a3fe35d3206e1e4261465d738aff93244ec90f2bd431a",
+    "zh:36a619af3767a92ee892c5de24604eeb9f23a5a01bb8455115a5eb4bd656f234",
+    "zh:45a374637b794427c5e07d23c6312d92d58bed3594789322c109d333ea1865e5",
+    "zh:538e501d313cfc0b61f3b2e5be9ae7755df3d3d9a3e4f14e0ea6a943d5102109",
+    "zh:64d8e4b94a1324292fe318bf27c6149aa345eabab8b89d9d78ce447ce5600e65",
+    "zh:7b3fcc0a724c5e00e6ce0e7da22010b6ae4bd2622544ef4d31fd4100f85985d7",
+    "zh:84876a614b010ae5dbef1b1edd9a22447cf57b9300b9eaf4321d587bfebf82dc",
+    "zh:850e3900fb2b55ad85b6def8b580fb851778bb470be5354cb0a0244d03acd5a4",
+    "zh:b6355d1eb7d165b246ad9c8f7c0ce7ccd5bbc58a01bd853c7ca896c71f4cd295",
+    "zh:bd4f1558f24af356d372937b810801555471eafbbc0552471bb6760f8ddd6b7e",
+    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
+    "zh:f78eaaf507ab56041112b765f6ca1740221773f3b32710bb8d087f29a686f30f",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/local" {
+  version     = "2.5.2"
+  constraints = "2.5.2"
+  hashes = [
+    "h1:JlMZD6nYqJ8sSrFfEAH0Vk/SL8WLZRmFaMUF9PJK5wM=",
+    "zh:136299545178ce281c56f36965bf91c35407c11897f7082b3b983d86cb79b511",
+    "zh:3b4486858aa9cb8163378722b642c57c529b6c64bfbfc9461d940a84cd66ebea",
+    "zh:4855ee628ead847741aa4f4fc9bed50cfdbf197f2912775dd9fe7bc43fa077c0",
+    "zh:4b8cd2583d1edcac4011caafe8afb7a95e8110a607a1d5fb87d921178074a69b",
+    "zh:52084ddaff8c8cd3f9e7bcb7ce4dc1eab00602912c96da43c29b4762dc376038",
+    "zh:71562d330d3f92d79b2952ffdda0dad167e952e46200c767dd30c6af8d7c0ed3",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:805f81ade06ff68fa8b908d31892eaed5c180ae031c77ad35f82cb7a74b97cf4",
+    "zh:8b6b3ebeaaa8e38dd04e56996abe80db9be6f4c1df75ac3cccc77642899bd464",
+    "zh:ad07750576b99248037b897de71113cc19b1a8d0bc235eb99173cc83d0de3b1b",
+    "zh:b9f1c3bfadb74068f5c205292badb0661e17ac05eb23bfe8bd809691e4583d0e",
+    "zh:cc4cbcd67414fefb111c1bf7ab0bc4beb8c0b553d01719ad17de9a047adff4d1",
+  ]
+}
diff --git a/infra/azure-peerpods-iam/main.tf b/infra/azure-peerpods-iam/main.tf
new file mode 100644
index 000000000..cab9d062b
--- /dev/null
+++ b/infra/azure-peerpods-iam/main.tf
@@ -0,0 +1,75 @@
+terraform {
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "4.5.0"
+    }
+    azuread = {
+      source  = "hashicorp/azuread"
+      version = "3.0.2"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = "2.5.2"
+    }
+  }
+}
+
+provider "azurerm" {
+  subscription_id = var.subscription_id
+  features {
+    resource_group {
+      prevent_deletion_if_contains_resources = false
+    }
+  }
+}
+
+data "azurerm_subscription" "current" {}
+
+data "azuread_client_config" "current" {}
+
+provider "azuread" {
+  tenant_id = data.azurerm_subscription.current.tenant_id
+}
+
+locals {
+  name = var.resource_group
+}
+
+resource "azurerm_resource_group" "rg" {
+  name     = var.resource_group
+  location = var.location
+}
+
+resource "azuread_application" "app" {
+  display_name = "${local.name}-app"
+  owners       = [data.azuread_client_config.current.object_id]
+}
+
+resource "azuread_service_principal" "sp" {
+  client_id                    = azuread_application.app.client_id
+  app_role_assignment_required = false
+  owners                       = [data.azuread_client_config.current.object_id]
+}
+
+resource "azurerm_role_assignment" "ra_vm_contributor" {
+  scope                = azurerm_resource_group.rg.id
+  role_definition_name = "Virtual Machine Contributor"
+  principal_id         = azuread_service_principal.sp.object_id
+}
+
+resource "azurerm_role_assignment" "ra_reader" {
+  scope                = azurerm_resource_group.rg.id
+  role_definition_name = "Reader"
+  principal_id         = azuread_service_principal.sp.object_id
+}
+
+resource "azurerm_role_assignment" "ra_network_contributor" {
+  scope                = azurerm_resource_group.rg.id
+  role_definition_name = "Network Contributor"
+  principal_id         = azuread_service_principal.sp.object_id
+}
+
+resource "azuread_application_password" "pw" {
+  application_id = azuread_application.app.id
+}
diff --git a/infra/azure-peerpods-iam/outs.tf b/infra/azure-peerpods-iam/outs.tf
new file mode 100644
index 000000000..64b692543
--- /dev/null
+++ b/infra/azure-peerpods-iam/outs.tf
@@ -0,0 +1,8 @@
+output "client_secret_env" {
+  value     = <<EOF
+client_id = "${azuread_application.app.client_id}"
+tenant_id = "${data.azurerm_subscription.current.tenant_id}"
+client_secret = "${azuread_application_password.pw.value}"
+EOF
+  sensitive = true
+}
diff --git a/infra/azure-peerpods-iam/vars.tf b/infra/azure-peerpods-iam/vars.tf
new file mode 100644
index 000000000..45b6eb3ec
--- /dev/null
+++ b/infra/azure-peerpods-iam/vars.tf
@@ -0,0 +1,11 @@
+variable "resource_group" {
+  type = string
+}
+
+variable "location" {
+  type = string
+}
+
+variable "subscription_id" {
+  type = string
+}
diff --git a/infra/azure-peerpods/.terraform.lock.hcl b/infra/azure-peerpods/.terraform.lock.hcl
index 2700567dc..cfc67815f 100644
--- a/infra/azure-peerpods/.terraform.lock.hcl
+++ b/infra/azure-peerpods/.terraform.lock.hcl
@@ -1,36 +1,6 @@
 # This file is maintained automatically by "terraform init".
 # Manual edits may be lost in future updates.
 
-provider "registry.terraform.io/hashicorp/azuread" {
-  version     = "3.0.2"
-  constraints = "3.0.2"
-  hashes = [
-    "h1:4HpBtur7h9Naz0BUhoJLVMQmmNABvpVDE/v/WC4LuHU=",
-    "h1:4ONsd+zmaW77NHdwY6tZ9f5Vk4uk5j4c6phuIAYd9c8=",
-    "h1:Ac2hOMzVtFxZL6U0znQB++O+AHsi47F4nZt0dGMAEJ4=",
-    "h1:HNrx7UJEDY5Kbx/r1LRQDWnziqvB6x3IU+pEA8Vq7dw=",
-    "h1:P807RV/+/XY1fylsKngmj2B5l2XOVTzqpd4ylZAwA+M=",
-    "h1:SDn/pi6q3CZyt9T8Bgobb91WziY2iE1teqr9Y8TlfJw=",
-    "h1:Sbb9HgPsFPsY3Jv8Kn+eoyYXoWHLWcODr7Okh/V001k=",
-    "h1:XUrQ/352oXVMh1ya8E7uMAmaC69zNICCIKqZ3kA4nXw=",
-    "h1:k0kPplqH7FWmnYeCXXrFIeCshgF1tC4LLhfk66bos3w=",
-    "h1:sYCyzbPpSYu2XDah8XqBUITQAfB0x4j4Twh6lw2C4CA=",
-    "h1:yQqvUtgtrYKGpIygdM8P6N+pvMWJJWIsVdPow29VE20=",
-    "zh:16e724b80a9004c7978c30f69a73c98ff63eb8a03937dd44c2a8f0ea0438b7a3",
-    "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7",
-    "zh:2bbbf13713ca4767267b889471c9fc14a56a8fdf5d1013da3ca78667e3caec64",
-    "zh:409ccb05431d643a079da082d89db2d95d6afed4769997ac537c8b7de3bff867",
-    "zh:53e4bca0f5d015380f7f524f36344afe6211ccaf614bfc69af73ca64a9f47d6c",
-    "zh:5780be2c1981d090604d7fa4cef675462f17f40e7f3dc501a031488e87a35b8f",
-    "zh:850e61a1b3e64c752c418526ccf48653514c861b36f5feb631619f906f7e99a0",
-    "zh:8c3565bfcea006a734149cc080452a9daf7d2a9d5362eb7e0a088b6c0d7f0f03",
-    "zh:908b9e6ad49d5d21173ecefc7924902047611be93bbf8e7d021aa9563358396f",
-    "zh:a2a79765c029bc58966eff61cb6e9b0ee14d2ac52b0a22fc7dfa35c9a49af669",
-    "zh:c7f56cbe8743e9ba81fce871bc97d9c07abe86770d9ee7ffefbf3882a61ba89a",
-    "zh:d4dba80e33421b30d81c62611fb7fc62ad39afecc6484436e635913cd8553e67",
-  ]
-}
-
 provider "registry.terraform.io/hashicorp/azurerm" {
   version     = "4.8.0"
   constraints = "4.8.0"
diff --git a/infra/azure-peerpods/main.tf b/infra/azure-peerpods/main.tf
index 37f89f3da..5ad79d332 100644
--- a/infra/azure-peerpods/main.tf
+++ b/infra/azure-peerpods/main.tf
@@ -4,10 +4,6 @@ terraform {
       source  = "hashicorp/azurerm"
       version = "4.8.0"
     }
-    azuread = {
-      source  = "hashicorp/azuread"
-      version = "3.0.2"
-    }
     local = {
       source  = "hashicorp/local"
       version = "2.5.2"
@@ -26,63 +22,16 @@ provider "azurerm" {
 
 data "azurerm_subscription" "current" {}
 
-data "azuread_client_config" "current" {}
-
-provider "azuread" {
-  tenant_id = data.azurerm_subscription.current.tenant_id
-}
-
 locals {
-  name = "${var.name_prefix}_caa_cluster"
+  name = "${var.name_prefix}contrast_pp"
 }
 
 data "azurerm_resource_group" "rg" {
-  name = local.name
-}
-
-resource "azuread_application" "app" {
-  display_name = local.name
-  owners       = [data.azuread_client_config.current.object_id]
-}
-
-resource "azuread_service_principal" "sp" {
-  client_id                    = azuread_application.app.client_id
-  app_role_assignment_required = false
-  owners                       = [data.azuread_client_config.current.object_id]
-}
-
-resource "azurerm_role_assignment" "ra_vm_contributor" {
-  scope                = data.azurerm_resource_group.rg.id
-  role_definition_name = "Virtual Machine Contributor"
-  principal_id         = azuread_service_principal.sp.object_id
-}
-
-resource "azurerm_role_assignment" "ra_reader" {
-  scope                = data.azurerm_resource_group.rg.id
-  role_definition_name = "Reader"
-  principal_id         = azuread_service_principal.sp.object_id
-}
-
-resource "azurerm_role_assignment" "ra_network_contributor" {
-  scope                = data.azurerm_resource_group.rg.id
-  role_definition_name = "Network Contributor"
-  principal_id         = azuread_service_principal.sp.object_id
-}
-
-resource "azuread_application_federated_identity_credential" "federated_credentials" {
-  display_name   = local.name
-  application_id = azuread_application.app.id
-  issuer         = azurerm_kubernetes_cluster.cluster.oidc_issuer_url
-  subject        = "system:serviceaccount:confidential-containers-system:cloud-api-adaptor"
-  audiences      = ["api://AzureADTokenExchange"]
-}
-
-resource "azuread_application_password" "cred" {
-  application_id = azuread_application.app.id
+  name = var.resource_group
 }
 
 resource "azurerm_virtual_network" "main" {
-  name                = local.name
+  name                = "${local.name}_net"
   address_space       = ["10.0.0.0/8"]
   location            = data.azurerm_resource_group.rg.location
   resource_group_name = data.azurerm_resource_group.rg.name
@@ -96,7 +45,7 @@ resource "azurerm_virtual_network" "main" {
 resource "azurerm_kubernetes_cluster" "cluster" {
   name                      = "${local.name}_aks"
   resource_group_name       = data.azurerm_resource_group.rg.name
-  node_resource_group       = "${local.name}_node_rg"
+  node_resource_group       = "${var.resource_group}_aks_node_rg"
   location                  = data.azurerm_resource_group.rg.location
   dns_prefix                = "aks"
   oidc_issuer_enabled       = true
@@ -128,31 +77,6 @@ resource "local_file" "kubeconfig" {
   content    = azurerm_kubernetes_cluster.cluster.kube_config_raw
 }
 
-resource "local_file" "workload_identity" {
-  filename        = "./workload-identity.yaml"
-  file_permission = "0777"
-  content         = <<EOF
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: cloud-api-adaptor-daemonset
-  namespace: confidential-containers-system
-spec:
-  template:
-    metadata:
-      labels:
-        azure.workload.identity/use: "true"
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: cloud-api-adaptor
-  namespace: confidential-containers-system
-  annotations:
-    azure.workload.identity/client-id: ${azuread_application.app.client_id}
-EOF
-}
-
 resource "local_file" "kustomization" {
   filename        = "./kustomization.yaml"
   file_permission = "0777"
@@ -178,6 +102,9 @@ configMapGenerator:
   - AZURE_RESOURCE_GROUP=${data.azurerm_resource_group.rg.name}
   - AZURE_SUBNET_ID=${one(azurerm_virtual_network.main.subnet.*.id)}
   - AZURE_IMAGE_ID=${var.image_id}
+  - AZURE_CLIENT_ID=${var.client_id}
+  - AZURE_TENANT_ID=${var.tenant_id}
+  - AZURE_CLIENT_SECRET=${var.client_secret}
   - DISABLECVM=false
 secretGenerator:
 - name: peer-pods-secret
@@ -186,7 +113,5 @@ secretGenerator:
   namespace: confidential-containers-system
   files:
   - id_rsa.pub
-patchesStrategicMerge:
-- workload-identity.yaml
 EOF
 }
diff --git a/infra/azure-peerpods/vars.tf b/infra/azure-peerpods/vars.tf
index e3f1185fd..570a8c76c 100644
--- a/infra/azure-peerpods/vars.tf
+++ b/infra/azure-peerpods/vars.tf
@@ -1,8 +1,9 @@
 variable "name_prefix" {
-  type = string
+  type    = string
+  default = ""
 }
 
-variable "image_resource_group_name" {
+variable "resource_group" {
   type = string
 }
 
@@ -10,6 +11,18 @@ variable "subscription_id" {
   type = string
 }
 
+variable "client_id" {
+  type = string
+}
+
+variable "tenant_id" {
+  type = string
+}
+
+variable "client_secret" {
+  type = string
+}
+
 variable "image_id" {
   type = string
 }
diff --git a/justfile b/justfile
index 4e4569459..58e10e903 100644
--- a/justfile
+++ b/justfile
@@ -54,7 +54,6 @@ node-installer platform=default_platform:
         "AKS-PEER-SNP")
             nix run -L .#scripts.deploy-caa -- \
                 --kustomization=./infra/azure-peerpods/kustomization.yaml \
-                --workload-identity=./infra/azure-peerpods/workload-identity.yaml \
                 --pub-key=./infra/azure-peerpods/id_rsa.pub
         ;;
         *)
@@ -176,11 +175,38 @@ undeploy:
     fi
 
 upload-image:
-    # Ensure that the resource group exists.
-    az group create  --name "${azure_resource_group}_caa_cluster" --location "$azure_location"
-
     nix run -L .#scripts.upload-image -- --subscription-id="$azure_subscription_id" --location="$azure_location" --resource-group="${azure_resource_group}_caa_cluster"
 
+# Create foundational dependencies of a CoCo-enabled cluster.
+create-pre platform=default_platform:
+    #!/usr/bin/env bash
+    set -euo pipefail
+    case {{ platform }} in
+        "AKS-CLH-SNP")
+            # TODO(burgerdev): this should create the resource group for consistency
+            :
+        ;;
+        "K3s-QEMU-SNP"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX")
+            :
+        ;;
+        "AKS-PEER-SNP")
+            echo "resource_group = \"${azure_resource_group}_caa_cluster\"" > infra/azure-peerpods-iam/just.auto.tfvars
+            echo "location = \"${azure_location}\"" >> infra/azure-peerpods-iam/just.auto.tfvars
+            echo "subscription_id = \"${azure_subscription_id}\"" >> infra/azure-peerpods-iam/just.auto.tfvars
+            nix run -L .#terraform -- -chdir=infra/azure-peerpods-iam init
+            nix run -L .#terraform -- -chdir=infra/azure-peerpods-iam apply --auto-approve
+            nix run -L .#terraform -- -chdir=infra/azure-peerpods-iam output -raw client_secret_env > infra/azure-peerpods/iam.auto.tfvars
+            echo "resource_group = \"${azure_resource_group}_caa_cluster\"" >> infra/azure-peerpods/iam.auto.tfvars
+
+            # TODO(burgerdev): this should be done in a generic upload target, together with OCI images
+            just upload-image
+        ;;
+        *)
+            echo "Unsupported platform: {{ platform }}"
+            exit 1
+        ;;
+    esac
+
 # Create a CoCo-enabled AKS cluster.
 create platform=default_platform:
     #!/usr/bin/env bash
@@ -193,15 +219,11 @@ create platform=default_platform:
             :
         ;;
         "AKS-PEER-SNP")
-            just upload-image
-
             # Populate Terraform variables.
-            echo "name_prefix = \"$azure_resource_group\"" > infra/azure-peerpods/just.auto.tfvars
-            echo "image_resource_group_name = \"$azure_resource_group\"" >> infra/azure-peerpods/just.auto.tfvars
-            echo "subscription_id = \"$azure_subscription_id\"" >> infra/azure-peerpods/just.auto.tfvars
+            echo "subscription_id = \"$azure_subscription_id\"" > infra/azure-peerpods/just.auto.tfvars
 
             nix run -L .#terraform -- -chdir=infra/azure-peerpods init
-            nix run -L .#terraform -- -chdir=infra/azure-peerpods apply
+            nix run -L .#terraform -- -chdir=infra/azure-peerpods apply --auto-approve
         ;;
         *)
             echo "Unsupported platform: {{ platform }}"
@@ -330,12 +352,35 @@ destroy platform=default_platform:
             :
         ;;
         "AKS-PEER-SNP")
-            nix run -L .#terraform -- -chdir=infra/azure-peerpods destroy
+            nix run -L .#terraform -- -chdir=infra/azure-peerpods destroy --auto-approve
 
             # Clean-up cached image ids.
             rm -f ${CONTRAST_CACHE_DIR}/image-upload/*.image-id
 
-            az group delete --name "${azure_resource_group}_caa_cluster" --yes
+        ;;
+        *)
+            echo "Unsupported platform: {{ platform }}"
+            exit 1
+        ;;
+    esac
+
+# Destroy foundational dependencies
+destroy-post platform=default_platform:
+    #!/usr/bin/env bash
+    set -euo pipefail
+    case {{ platform }} in
+        "AKS-CLH-SNP")
+            # TODO(burgerdev): this should destroy the resource group for consistency.
+            :
+        ;;
+        "K3s-QEMU-SNP"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX")
+            :
+        ;;
+        "AKS-PEER-SNP")
+            nix run -L .#terraform -- -chdir=infra/azure-peerpods-iam destroy --auto-approve
+
+            # We just destroyed the resource group, so these IDs are invalid.
+            rm -f ${CONTRAST_CACHE_DIR}/image-upload/*.image-id
         ;;
         *)
             echo "Unsupported platform: {{ platform }}"
diff --git a/packages/scripts.nix b/packages/scripts.nix
index 92d9934ee..22fabeee4 100644
--- a/packages/scripts.nix
+++ b/packages/scripts.nix
@@ -466,10 +466,6 @@
           kustomizationFile="''${i#*=}"
           shift
           ;;
-        --workload-identity=*)
-          workloadIdentityFile="''${i#*=}"
-          shift
-          ;;
         --pub-key=*)
           pubKeyFile="''${i#*=}"
           shift
@@ -485,7 +481,6 @@
       cp -r ${pkgs.cloud-api-adaptor.src}/src/cloud-api-adaptor/install/* "$tmpdir"
       chmod -R +w "$tmpdir"
       cp "$kustomizationFile" "$tmpdir/overlays/azure/kustomization.yaml"
-      cp "$workloadIdentityFile" "$tmpdir/overlays/azure/workload-identity.yaml"
       cp "$pubKeyFile" "$tmpdir/overlays/azure/id_rsa.pub"
 
       kubectl apply -k "github.com/confidential-containers/operator/config/release?ref=v${pkgs.cloud-api-adaptor.version}"

From 3e96b98e619fda87f1c89d25871e23fb941dee46 Mon Sep 17 00:00:00 2001
From: Markus Rudy <mr@edgeless.systems>
Date: Fri, 15 Nov 2024 15:55:22 +0100
Subject: [PATCH 4/4] e2e: smoke test for peer pods

Co-authored-by: Paul Meyer <49727155+katexochen@users.noreply.github.com>
---
 .github/workflows/e2e_peerpods.yml | 50 ++++++++++++++++++++++++++
 packages/scripts.nix               | 10 ++++++
 packages/test-peerpods.sh          | 58 ++++++++++++++++++++++++++++++
 3 files changed, 118 insertions(+)
 create mode 100644 .github/workflows/e2e_peerpods.yml
 create mode 100644 packages/test-peerpods.sh

diff --git a/.github/workflows/e2e_peerpods.yml b/.github/workflows/e2e_peerpods.yml
new file mode 100644
index 000000000..9b0c5cf53
--- /dev/null
+++ b/.github/workflows/e2e_peerpods.yml
@@ -0,0 +1,50 @@
+name: e2e peer-pods
+
+on:
+  workflow_dispatch:
+    inputs:
+      image-id:
+        description: "ID of the guest VM image to test (default: build a fresh image)"
+        required: false
+  pull_request:
+    paths:
+      - .github/workflows/e2e_peerpods.yml
+      - packages/test-peerpods.sh
+      - packages/by-name/cloud-api-adaptor/**
+      - packages/by-name/kata/**
+      - packages/by-name/image-podvm/**
+      - packages/nixos
+
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: ./.github/actions/setup_nix
+        with:
+          githubToken: ${{ secrets.GITHUB_TOKEN }}
+          cachixToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
+      - name: Login to Azure
+        uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0
+        with:
+          creds: ${{ secrets.CONTRAST_CI_INFRA_AZURE }}
+      - name: Test peer-pods
+        env:
+          azure_subscription_id: ${{ vars.AZURE_SUBSCRIPTION_ID }}
+          azure_image_id: ${{ inputs.image-id }}
+          azure_resource_group: contrast-ci
+          azure_location: germanywestcentral
+          CONTRAST_CACHE_DIR: "./workspace.cache"
+        run: |
+          ssh-keygen -t rsa -f ./infra/azure-peerpods/id_rsa -N ""
+          cat >infra/azure-peerpods/iam.auto.tfvars <<EOF
+          tenant_id = "${{ vars.AZURE_TENANT_ID }}"
+          client_id = "${{ vars.PEER_POD_CLIENT_ID_AZURE }}"
+          client_secret = "${{ secrets.PEER_POD_CLIENT_SECRET_AZURE }}"
+          resource_group = "contrast-ci"
+          EOF
+          nix run .#scripts.test-peerpods
+      - name: Terminate cluster
+        if: always()
+        run: |
+          nix run -L .#terraform -- -chdir=infra/azure-peerpods destroy --auto-approve
diff --git a/packages/scripts.nix b/packages/scripts.nix
index 22fabeee4..908556ab3 100644
--- a/packages/scripts.nix
+++ b/packages/scripts.nix
@@ -509,4 +509,14 @@
     ];
     text = builtins.readFile ./cleanup-images.sh;
   };
+
+  test-peerpods = writeShellApplication {
+    name = "test-peerpods";
+    runtimeInputs = with pkgs; [
+      just
+      azure-cli
+      kubectl
+    ];
+    text = builtins.readFile ./test-peerpods.sh;
+  };
 }
diff --git a/packages/test-peerpods.sh b/packages/test-peerpods.sh
new file mode 100644
index 000000000..e7ca1a494
--- /dev/null
+++ b/packages/test-peerpods.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# Copyright 2024 Edgeless Systems GmbH
+# SPDX-License-Identifier: AGPL-3.0-only
+
+set -euo pipefail
+
+set -x
+
+if [ -z "${azure_image_id}" ]; then
+  nix run -L .#scripts.upload-image -- \
+    --subscription-id="${azure_subscription_id:?}" \
+    --location="${azure_location:?}" \
+    --resource-group="${azure_resource_group:?}"
+else
+  echo "image_id = \"${azure_image_id}\"" >infra/azure-peerpods/image_id.auto.tfvars
+fi
+
+cat >infra/azure-peerpods/e2e.auto.tfvars <<EOF
+name_prefix = "${azure_resource_group:?}-$RANDOM-"
+EOF
+
+just create AKS-PEER-SNP
+just get-credentials AKS-PEER-SNP
+just node-installer AKS-PEER-SNP
+
+cleanup() {
+  kubectl delete deploy nginx
+  kubectl wait --for=delete pod --selector=app=nginx --timeout=5m
+}
+
+trap cleanup EXIT
+
+kubectl apply -f - <<EOF
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+spec:
+  selector:
+    matchLabels:
+      app: nginx
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      runtimeClassName: kata-remote
+      containers:
+      - name: nginx
+        image: nginx
+        imagePullPolicy: Always
+EOF
+
+if ! kubectl wait --for=condition=available --timeout=5m deployment/nginx; then
+  kubectl describe pods
+  exit 1
+fi