From 933caf31ab0a2d9673d0bc5f93f45cc4ed3386eb Mon Sep 17 00:00:00 2001 From: Paul Meyer <49727155+katexochen@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:02:20 +0100 Subject: [PATCH 01/19] Revert "just: remove cases with ns.yml file" This reverts commit 03e2b6f69520eb3ff801003162364a708d64e233. --- justfile | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/justfile b/justfile index 3275e4e31..4f2c47e39 100644 --- a/justfile +++ b/justfile @@ -142,6 +142,14 @@ apply-runtime target=default_deploy_target platform=default_platform: apply target=default_deploy_target: #!/usr/bin/env bash set -euo pipefail + case {{ target }} in + "openssl" | "emojivoto" | "volume-stateful-set") + : + ;; + *) + kubectl apply -f ./{{ workspace_dir }}/deployment/ns.yml + ;; + esac kubectl apply -f ./{{ workspace_dir }}/deployment # Delete Kubernetes manifests. @@ -167,7 +175,15 @@ undeploy platform=default_platform: --cascade=foreground \ --ignore-not-found fi - kubectl delete namespace $ns + if [[ -f ./{{ workspace_dir }}/deployment/ns.yml ]]; then + kubectl delete \ + -f ./{{ workspace_dir }}/deployment \ + --ignore-not-found \ + --grace-period=30 \ + --timeout=10m + else + kubectl delete namespace $ns + fi upload-image: # Ensure that the resource group exists. From db557a35b0f0f44232e900ddd598851b1cd9ea4c Mon Sep 17 00:00:00 2001 From: Paul Meyer <49727155+katexochen@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:02:21 +0100 Subject: [PATCH 02/19] Revert "just: ensure peerpod vms are cleaned up on undeploy" This reverts commit 80c197bd85383c76d48f0cf74081b4e4dfb48c00. --- justfile | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/justfile b/justfile index 4f2c47e39..6f642dc9d 100644 --- a/justfile +++ b/justfile @@ -1,5 +1,5 @@ # Undeploy, rebuild, deploy. -default target=default_deploy_target platform=default_platform cli=default_cli: (soft-clean platform) coordinator initializer openssl cryptsetup port-forwarder service-mesh-proxy (node-installer platform) (deploy target cli platform) set verify (wait-for-workload target) +default target=default_deploy_target platform=default_platform cli=default_cli: soft-clean coordinator initializer openssl cryptsetup port-forwarder service-mesh-proxy (node-installer platform) (deploy target cli platform) set verify (wait-for-workload target) # Build and push a container image. push target: @@ -62,7 +62,7 @@ node-installer platform=default_platform: ;; esac -e2e target=default_deploy_target platform=default_platform: (soft-clean platform) coordinator initializer cryptsetup openssl port-forwarder service-mesh-proxy (node-installer platform) +e2e target=default_deploy_target platform=default_platform: soft-clean coordinator initializer cryptsetup openssl port-forwarder service-mesh-proxy (node-installer platform) #!/usr/bin/env bash set -euo pipefail if [[ "{{ target }}" == "aks-runtime" ]]; then @@ -153,7 +153,7 @@ apply target=default_deploy_target: kubectl apply -f ./{{ workspace_dir }}/deployment # Delete Kubernetes manifests. -undeploy platform=default_platform: +undeploy: #!/usr/bin/env bash set -euo pipefail if [[ ! -d ./{{ workspace_dir }} ]]; then @@ -169,12 +169,6 @@ undeploy platform=default_platform: echo "Namespace $ns does not exist, nothing to undeploy." exit 0 fi - if [[ {{ platform }} == "AKS-PEER-SNP" ]]; then - kubectl delete \ - -f ./{{ workspace_dir }}/deployment \ - --cascade=foreground \ - --ignore-not-found - fi if [[ -f ./{{ workspace_dir }}/deployment/ns.yml ]]; then kubectl delete \ -f ./{{ workspace_dir }}/deployment \ @@ -365,18 +359,18 @@ fmt: lint: nix run -L .#scripts.golangci-lint -- run -demodir version="latest" platform=default_platform: (undeploy platform) +demodir version="latest": undeploy #!/usr/bin/env bash set -euo pipefail v="$(echo {{ version }} | sed 's/\./-/g')" nix develop -u DIRENV_DIR -u DIRENV_FILE -u DIRENV_DIFF .#demo-$v # Remove deployment specific files. -soft-clean platform=default_platform: (undeploy platform) +soft-clean: undeploy rm -rf ./{{ workspace_dir }} # Cleanup all auxiliary files, caches etc. -clean platform=default_platform: (soft-clean platform) +clean: soft-clean rm -rf ./{{ workspace_dir }}.cache rm -rf ./layers_cache rm -f ./layers-cache.json From 85fe4dc1afbc3f2649ef3343770ff7c3eb1ffb99 Mon Sep 17 00:00:00 2001 From: Paul Meyer <49727155+katexochen@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:02:28 +0100 Subject: [PATCH 03/19] Revert "kuberesource: use our own CAA image" This reverts commit 3fb6a9dee1c5178293467ed4ea91801e6eef5424. --- internal/kuberesource/parts.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/kuberesource/parts.go b/internal/kuberesource/parts.go index a7f324b4e..13b999536 100644 --- a/internal/kuberesource/parts.go +++ b/internal/kuberesource/parts.go @@ -131,11 +131,12 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle cloudAPIAdaptor := Container(). WithName("cloud-api-adaptor"). - WithImage("ghcr.io/edgelesssys/contrast/cloud-api-adaptor:latest"). + // TODO(freax13): Don't hard-code this + WithImage("quay.io/confidential-containers/cloud-api-adaptor:v0.9.0-amd64"). WithVolumeMounts( VolumeMount(). WithName("ssh"). - WithMountPath("/.ssh/"). + WithMountPath("/root/.ssh/"). WithReadOnly(true), VolumeMount(). WithName("pods-dir"). @@ -144,10 +145,9 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle WithName("netns"). WithMountPath("/run/netns"). WithMountPropagation(corev1.MountPropagationHostToContainer), - VolumeMount(). - WithName("netns"). - WithMountPath("/var/run/netns"). - WithMountPropagation(corev1.MountPropagationHostToContainer), + ). + WithArgs( + "/usr/local/bin/entrypoint.sh", ). WithEnv( NewEnvVar("optionals", fmt.Sprintf("-socket /run/peerpod/hypervisor-%s.sock ", runtimeHandler)), From a475b4a04b71f9330b2e54baed09674ea7ccd0b8 Mon Sep 17 00:00:00 2001 From: Paul Meyer <49727155+katexochen@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:02:29 +0100 Subject: [PATCH 04/19] Revert "justfile: push cloud-api-adaptor" This reverts commit 3ae4e073e0ef7b449f670748cd0c509ac33cb797. --- justfile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/justfile b/justfile index 6f642dc9d..887cb6271 100644 --- a/justfile +++ b/justfile @@ -47,14 +47,9 @@ node-installer platform=default_platform: just push "tardev-snapshotter" just push "node-installer-microsoft" ;; - "K3s-QEMU-SNP"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX") - just push "nydus-snapshotter" - just push "node-installer-kata" - ;; - "AKS-PEER-SNP") + "AKS-PEER-SNP"|"K3s-QEMU-SNP"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX") just push "nydus-snapshotter" just push "node-installer-kata" - just push "cloud-api-adaptor" ;; *) echo "Unsupported platform: {{ platform }}" From 0103070f695428b653b193ad8da7f25e703763bc Mon Sep 17 00:00:00 2001 From: Paul Meyer <49727155+katexochen@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:02:35 +0100 Subject: [PATCH 05/19] Revert "infra/azure-peerpods: remove unnecessary resources" This reverts commit 56fc63daf580cf0ac801fe869f7e62f7724a87c4. --- .gitignore | 2 + infra/azure-peerpods/main.tf | 72 ++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/.gitignore b/.gitignore index 3bb208658..f887b78b7 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,6 @@ id_rsa* kube.conf out.env infra/**/peer-pods-config.yaml +infra/**/kustomization.yaml +infra/**/workload-identity.yaml uplosi.conf* diff --git a/infra/azure-peerpods/main.tf b/infra/azure-peerpods/main.tf index 1f565a533..c77a806e4 100644 --- a/infra/azure-peerpods/main.tf +++ b/infra/azure-peerpods/main.tf @@ -69,6 +69,14 @@ resource "azurerm_role_assignment" "ra_network_contributor" { principal_id = azuread_service_principal.sp.object_id } +resource "azuread_application_federated_identity_credential" "federated_credentials" { + display_name = local.name + application_id = azuread_application.app.id + issuer = azurerm_kubernetes_cluster.cluster.oidc_issuer_url + subject = "system:serviceaccount:confidential-containers-system:cloud-api-adaptor" + audiences = ["api://AzureADTokenExchange"] +} + resource "azuread_application_password" "cred" { application_id = azuread_application.app.id } @@ -120,6 +128,70 @@ resource "local_file" "kubeconfig" { content = azurerm_kubernetes_cluster.cluster.kube_config_raw } +resource "local_file" "workload_identity" { + filename = "./workload-identity.yaml" + file_permission = "0777" + content = < Date: Tue, 12 Nov 2024 09:02:36 +0100 Subject: [PATCH 06/19] Revert "scripts: remove deploy-caa" This reverts commit 5aed8d806bf31c093def2b9121ed02c7fdf92da1. --- packages/scripts.nix | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/packages/scripts.nix b/packages/scripts.nix index 6c4a7c02f..02d762ad9 100644 --- a/packages/scripts.nix +++ b/packages/scripts.nix @@ -452,6 +452,46 @@ ''; }; + deploy-caa = writeShellApplication { + name = "deploy-caa"; + runtimeInputs = with pkgs; [ kubectl ]; + text = '' + set -euo pipefail + + for i in "$@"; do + case $i in + --kustomization=*) + kustomizationFile="''${i#*=}" + shift + ;; + --workload-identity=*) + workloadIdentityFile="''${i#*=}" + shift + ;; + --pub-key=*) + pubKeyFile="''${i#*=}" + shift + ;; + *) + echo "Unknown option $i" + exit 1 + ;; + esac + done + + tmpdir=$(mktemp -d) + cp -r ${pkgs.cloud-api-adaptor.src}/src/cloud-api-adaptor/install/* "$tmpdir" + chmod -R +w "$tmpdir" + cp "$kustomizationFile" "$tmpdir/overlays/azure/kustomization.yaml" + cp "$workloadIdentityFile" "$tmpdir/overlays/azure/workload-identity.yaml" + cp "$pubKeyFile" "$tmpdir/overlays/azure/id_rsa.pub" + + kubectl apply -k "github.com/confidential-containers/operator/config/release?ref=v${pkgs.cloud-api-adaptor.version}" + kubectl apply -k "github.com/confidential-containers/operator/config/samples/ccruntime/peer-pods?ref=v${pkgs.cloud-api-adaptor.version}" + kubectl apply -k "$tmpdir/overlays/azure" + ''; + }; + cleanup-bare-metal = writeShellApplication { name = "cleanup-bare-metal"; runtimeInputs = with pkgs; [ From ee335a7b806a617cb345ced4be1c7eb1f68e45e3 Mon Sep 17 00:00:00 2001 From: Paul Meyer <49727155+katexochen@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:02:36 +0100 Subject: [PATCH 07/19] Revert "justfile: use node-installer instead of deploy-caa" This reverts commit 2371db38e2fe3af1edf40db30a8338446443cd4c. --- justfile | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/justfile b/justfile index 887cb6271..4e4569459 100644 --- a/justfile +++ b/justfile @@ -47,10 +47,16 @@ node-installer platform=default_platform: just push "tardev-snapshotter" just push "node-installer-microsoft" ;; - "AKS-PEER-SNP"|"K3s-QEMU-SNP"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX") + "K3s-QEMU-SNP"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX") just push "nydus-snapshotter" just push "node-installer-kata" ;; + "AKS-PEER-SNP") + nix run -L .#scripts.deploy-caa -- \ + --kustomization=./infra/azure-peerpods/kustomization.yaml \ + --workload-identity=./infra/azure-peerpods/workload-identity.yaml \ + --pub-key=./infra/azure-peerpods/id_rsa.pub + ;; *) echo "Unsupported platform: {{ platform }}" exit 1 @@ -70,7 +76,7 @@ e2e target=default_deploy_target platform=default_platform: soft-clean coordinat --skip-undeploy=true # Generate policies, apply Kubernetes manifests. -deploy target=default_deploy_target cli=default_cli platform=default_platform: (runtime target platform) (apply-runtime target platform) (populate target platform) (generate cli platform) (apply target) +deploy target=default_deploy_target cli=default_cli platform=default_platform: (runtime target platform) (apply "runtime") (populate target platform) (generate cli platform) (apply target) # Populate the workspace with a runtime class deployment runtime target=default_deploy_target platform=default_platform: @@ -124,20 +130,15 @@ generate cli=default_cli platform=default_platform: ;; esac -# Apply the runtime. -apply-runtime target=default_deploy_target platform=default_platform: - #!/usr/bin/env bash - set -euo pipefail - kubectl apply -f ./{{ workspace_dir }}/runtime - if [[ {{ platform }} == "AKS-PEER-SNP" ]]; then - kubectl apply -f ./infra/azure-peerpods/peer-pods-config.yaml --namespace {{ target }}${namespace_suffix-} - fi - # Apply Kubernetes manifests from /deployment apply target=default_deploy_target: #!/usr/bin/env bash set -euo pipefail case {{ target }} in + "runtime") + kubectl apply -f ./{{ workspace_dir }}/runtime + exit 0 + ;; "openssl" | "emojivoto" | "volume-stateful-set") : ;; From fde72c2f0f0a745e075a64cb6a29845a85ea4156 Mon Sep 17 00:00:00 2001 From: Paul Meyer <49727155+katexochen@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:02:37 +0100 Subject: [PATCH 08/19] Revert "infra/azure-peerpods: create k8s resources for configuring CAA" This reverts commit aeff4290b0b23686e04d2d53871d0948b2b5e26b. --- .gitignore | 1 - infra/azure-peerpods/main.tf | 44 ------------------------------------ 2 files changed, 45 deletions(-) diff --git a/.gitignore b/.gitignore index f887b78b7..3f764fc51 100644 --- a/.gitignore +++ b/.gitignore @@ -21,7 +21,6 @@ terraform.tfstate* id_rsa* kube.conf out.env -infra/**/peer-pods-config.yaml infra/**/kustomization.yaml infra/**/workload-identity.yaml uplosi.conf* diff --git a/infra/azure-peerpods/main.tf b/infra/azure-peerpods/main.tf index c77a806e4..37f89f3da 100644 --- a/infra/azure-peerpods/main.tf +++ b/infra/azure-peerpods/main.tf @@ -190,47 +190,3 @@ patchesStrategicMerge: - workload-identity.yaml EOF } - - -data "local_file" "id_rsa" { - filename = "id_rsa.pub" -} - -resource "local_file" "peer-pods-config" { - filename = "./peer-pods-config.yaml" - file_permission = "0777" - content = < Date: Tue, 12 Nov 2024 09:02:38 +0100 Subject: [PATCH 09/19] Revert "contrast: add empty reference values for AKS-PEER-SNP" This reverts commit c93d6b8117d79534c38a8f2538c8226fd523dca0. --- packages/by-name/contrast/package.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/by-name/contrast/package.nix b/packages/by-name/contrast/package.nix index 45f17141b..6eacad85f 100644 --- a/packages/by-name/contrast/package.nix +++ b/packages/by-name/contrast/package.nix @@ -52,7 +52,6 @@ let k3s-qemu-tdx-handler = runtimeHandler "k3s-qemu-tdx" kata.contrast-node-installer-image.runtimeHash; rke2-qemu-tdx-handler = runtimeHandler "rke2-qemu-tdx" kata.contrast-node-installer-image.runtimeHash; k3s-qemu-snp-handler = runtimeHandler "k3s-qemu-snp" kata.contrast-node-installer-image.runtimeHash; - aks-peer-snp-handler = runtimeHandler "aks-peer-snp" kata.contrast-node-installer-image.runtimeHash; aksRefVals = { snp = [ @@ -129,7 +128,6 @@ let "${k3s-qemu-tdx-handler}" = tdxRefVals; "${rke2-qemu-tdx-handler}" = tdxRefVals; "${k3s-qemu-snp-handler}" = snpRefVals; - "${aks-peer-snp-handler}" = { }; } ); From 36c6293e74c81b2c947a7e0cd26cd2081a33766f Mon Sep 17 00:00:00 2001 From: Paul Meyer <49727155+katexochen@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:02:39 +0100 Subject: [PATCH 10/19] Revert "kuberesource: include CAA in node-installer pod on AKS-PEER-SNP" This reverts commit 18dca4be37c1074f961b9177defd0a55037ec582. --- internal/kuberesource/parts.go | 85 ++++------------------------------ 1 file changed, 8 insertions(+), 77 deletions(-) diff --git a/internal/kuberesource/parts.go b/internal/kuberesource/parts.go index 13b999536..fad37fc45 100644 --- a/internal/kuberesource/parts.go +++ b/internal/kuberesource/parts.go @@ -129,87 +129,18 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle ), } - cloudAPIAdaptor := Container(). - WithName("cloud-api-adaptor"). - // TODO(freax13): Don't hard-code this - WithImage("quay.io/confidential-containers/cloud-api-adaptor:v0.9.0-amd64"). - WithVolumeMounts( - VolumeMount(). - WithName("ssh"). - WithMountPath("/root/.ssh/"). - WithReadOnly(true), - VolumeMount(). - WithName("pods-dir"). - WithMountPath("/run/peerpod"), - VolumeMount(). - WithName("netns"). - WithMountPath("/run/netns"). - WithMountPropagation(corev1.MountPropagationHostToContainer), - ). - WithArgs( - "/usr/local/bin/entrypoint.sh", - ). - WithEnv( - NewEnvVar("optionals", fmt.Sprintf("-socket /run/peerpod/hypervisor-%s.sock ", runtimeHandler)), - ). - WithEnvFrom( - applycorev1.EnvFromSource(). - WithConfigMapRef( - applycorev1.ConfigMapEnvSource(). - WithName("peer-pods-cm"), - ), - applycorev1.EnvFromSource(). - WithSecretRef(applycorev1.SecretEnvSource(). - WithName("azure-client-secret"), - ), - ). - WithSecurityContext( - applycorev1.SecurityContext(). - WithCapabilities( - applycorev1.Capabilities(). - WithAdd( - corev1.Capability("NET_ADMIN"), - corev1.Capability("SYS_ADMIN"), - ), - ), - ) - cloudAPIAdaptorVolumes := []*applycorev1.VolumeApplyConfiguration{ - Volume(). - WithName("pods-dir"). - WithHostPath(HostPathVolumeSource(). - WithPath("/run/peerpod"). - WithType(corev1.HostPathDirectoryOrCreate), - ), - Volume(). - WithName("netns"). - WithHostPath(HostPathVolumeSource(). - WithPath("/run/netns"). - WithType(corev1.HostPathDirectory), - ), - Volume(). - WithName("ssh"). - WithSecret(applycorev1.SecretVolumeSource(). - WithDefaultMode(0o600). - WithSecretName("ssh-key-secret"), - ), - } - var nodeInstallerImageURL string - var containers []*applycorev1.ContainerApplyConfiguration - var volumes []*applycorev1.VolumeApplyConfiguration + var snapshotter *applycorev1.ContainerApplyConfiguration + var snapshotterVolumes []*applycorev1.VolumeApplyConfiguration switch platform { case platforms.AKSCloudHypervisorSNP: nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-microsoft:latest" - containers = []*applycorev1.ContainerApplyConfiguration{tardevSnapshotter} - volumes = tardevSnapshotterVolumes + snapshotter = tardevSnapshotter + snapshotterVolumes = tardevSnapshotterVolumes case platforms.K3sQEMUTDX, platforms.K3sQEMUSNP, platforms.RKE2QEMUTDX: nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata:latest" - containers = []*applycorev1.ContainerApplyConfiguration{nydusSnapshotter} - volumes = nydusSnapshotterVolumes - case platforms.AKSPeerSNP: - nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata:latest" - containers = []*applycorev1.ContainerApplyConfiguration{nydusSnapshotter, cloudAPIAdaptor} - volumes = append(nydusSnapshotterVolumes, cloudAPIAdaptorVolumes...) + snapshotter = nydusSnapshotter + snapshotterVolumes = nydusSnapshotterVolumes default: return nil, fmt.Errorf("unsupported platform %q", platform) } @@ -241,10 +172,10 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle WithCommand("/bin/node-installer", platform.String()), ). WithContainers( - containers..., + snapshotter, ). WithVolumes(append( - volumes, + snapshotterVolumes, Volume(). WithName("host-mount"). WithHostPath(HostPathVolumeSource(). From 39e82748ee5ab55f4c7e00ef12e5b0bfaecf7fbb Mon Sep 17 00:00:00 2001 From: Paul Meyer <49727155+katexochen@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:02:39 +0100 Subject: [PATCH 11/19] Revert "kuberesource: use different containerd path for AKS-PEER-SNP" This reverts commit 02715a1c7cdc3edf2250a62afc8e25cc62d64f30. --- internal/kuberesource/parts.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/internal/kuberesource/parts.go b/internal/kuberesource/parts.go index fad37fc45..056aadb95 100644 --- a/internal/kuberesource/parts.go +++ b/internal/kuberesource/parts.go @@ -82,11 +82,6 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle ), } - containerdPath := "/var/lib/rancher/k3s/agent/containerd" - if platform == platforms.AKSPeerSNP { - containerdPath = "/var/lib/containerd" - } - nydusSnapshotter := Container(). WithName("nydus-snapshotter"). WithImage("ghcr.io/edgelesssys/contrast/nydus-snapshotter:latest"). @@ -118,7 +113,7 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle Volume(). WithName("var-lib-containerd"). WithHostPath(HostPathVolumeSource(). - WithPath(containerdPath). + WithPath("/var/lib/rancher/k3s/agent/containerd"). WithType(corev1.HostPathDirectory), ), Volume(). From 3ecbce3e101152ffdebd097444947ef604d180b4 Mon Sep 17 00:00:00 2001 From: Paul Meyer <49727155+katexochen@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:02:40 +0100 Subject: [PATCH 12/19] Revert "node-installer: support AKS-PEER-SNP when patching containerd config" This reverts commit 7a16894ba3806a1fe8e1f60a80ea64355d2bd8e5. --- nodeinstaller/node-installer.go | 7 +- nodeinstaller/node-installer_test.go | 6 -- .../testdata/expected-aks-peer-snp.toml | 81 ------------------- 3 files changed, 2 insertions(+), 92 deletions(-) delete mode 100644 nodeinstaller/testdata/expected-aks-peer-snp.toml diff --git a/nodeinstaller/node-installer.go b/nodeinstaller/node-installer.go index 268f7a8e5..5f38bddef 100644 --- a/nodeinstaller/node-installer.go +++ b/nodeinstaller/node-installer.go @@ -107,9 +107,6 @@ func run(ctx context.Context, fetcher assetFetcher, platform platforms.Platform, case platforms.AKSCloudHypervisorSNP: kataConfigPath = filepath.Join(kataConfigPath, "configuration-clh-snp.toml") containerdConfigPath = filepath.Join(hostMount, "etc", "containerd", "config.toml") - case platforms.AKSPeerSNP: - kataConfigPath = filepath.Join(kataConfigPath, "configuration-peer-snp.toml") - containerdConfigPath = filepath.Join(hostMount, "etc", "containerd", "config.toml") case platforms.K3sQEMUSNP: kataConfigPath = filepath.Join(kataConfigPath, "configuration-qemu-snp.toml") containerdConfigPath = filepath.Join(hostMount, "var", "lib", "rancher", "k3s", "agent", "etc", "containerd", "config.toml.tmpl") @@ -142,7 +139,7 @@ func run(ctx context.Context, fetcher assetFetcher, platform platforms.Platform, } switch platform { - case platforms.AKSCloudHypervisorSNP, platforms.AKSPeerSNP: + case platforms.AKSCloudHypervisorSNP: return restartHostContainerd(containerdConfigPath, "containerd") case platforms.K3sQEMUTDX, platforms.K3sQEMUSNP: if hostServiceExists("k3s") { @@ -209,7 +206,7 @@ func patchContainerdConfig(runtimeHandler, basePath, configPath string, platform case platforms.AKSCloudHypervisorSNP: snapshotterName = fmt.Sprintf("tardev-%s", runtimeHandler) socketName = fmt.Sprintf("/run/containerd/tardev-snapshotter-%s.sock", runtimeHandler) - case platforms.K3sQEMUTDX, platforms.K3sQEMUSNP, platforms.RKE2QEMUTDX, platforms.AKSPeerSNP: + case platforms.K3sQEMUTDX, platforms.K3sQEMUSNP, platforms.RKE2QEMUTDX: snapshotterName = fmt.Sprintf("nydus-%s", runtimeHandler) socketName = fmt.Sprintf("/run/containerd/containerd-nydus-grpc-%s.sock", runtimeHandler) diff --git a/nodeinstaller/node-installer_test.go b/nodeinstaller/node-installer_test.go index 57a692665..79f80ec34 100644 --- a/nodeinstaller/node-installer_test.go +++ b/nodeinstaller/node-installer_test.go @@ -18,8 +18,6 @@ import ( var ( //go:embed testdata/expected-aks-clh-snp.toml expectedConfAKSCLHSNP []byte - //go:embed testdata/expected-aks-peer-snp.toml - expectedConfAKSPeerSNP []byte //go:embed testdata/expected-bare-metal-qemu-tdx.toml expectedConfBareMetalQEMUTDX []byte //go:embed testdata/expected-bare-metal-qemu-snp.toml @@ -36,10 +34,6 @@ func TestPatchContainerdConfig(t *testing.T) { platform: platforms.AKSCloudHypervisorSNP, expected: expectedConfAKSCLHSNP, }, - "AKSPeerSNP": { - platform: platforms.AKSPeerSNP, - expected: expectedConfAKSPeerSNP, - }, "BareMetalQEMUTDX": { platform: platforms.K3sQEMUTDX, expected: expectedConfBareMetalQEMUTDX, diff --git a/nodeinstaller/testdata/expected-aks-peer-snp.toml b/nodeinstaller/testdata/expected-aks-peer-snp.toml deleted file mode 100644 index 5506b7878..000000000 --- a/nodeinstaller/testdata/expected-aks-peer-snp.toml +++ /dev/null @@ -1,81 +0,0 @@ -version = 2 - -[debug] -level = 'debug' - -[metrics] -address = '0.0.0.0:10257' - -[plugins] -[plugins.'io.containerd.grpc.v1.cri'] -sandbox_image = 'mcr.microsoft.com/oss/kubernetes/pause:3.6' - -[plugins.'io.containerd.grpc.v1.cri'.cni] -bin_dir = '/opt/cni/bin' -conf_dir = '/etc/cni/net.d' -conf_template = '/etc/containerd/kubenet_template.conf' - -[plugins.'io.containerd.grpc.v1.cri'.containerd] -default_runtime_name = 'runc' -disable_snapshot_annotations = false -discard_unpacked_layers = false - -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes] -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.kata] -runtime_type = 'io.containerd.kata.v2' - -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.kata-cc] -pod_annotations = ['io.katacontainers.*'] -privileged_without_host_devices = true -runtime_type = 'io.containerd.kata-cc.v2' -snapshotter = 'tardev' - -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.kata-cc.options] -ConfigPath = '/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml' - -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.katacli] -runtime_type = 'io.containerd.runc.v1' - -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.katacli.options] -BinaryName = '/usr/bin/kata-runtime' -CriuPath = '' -IoGid = 0 -IoUid = 0 -NoNewKeyring = false -NoPivotRoot = false -Root = '' -ShimCgroup = '' -SystemdCgroup = false - -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.my-runtime] -runtime_type = 'io.containerd.contrast-cc.v2' -runtime_path = '/opt/edgeless/my-runtime/bin/containerd-shim-contrast-cc-v2' -pod_annotations = ['io.katacontainers.*'] -privileged_without_host_devices = true -snapshotter = 'nydus-my-runtime' - -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.my-runtime.options] -ConfigPath = '/opt/edgeless/my-runtime/etc/configuration-peer-snp.toml' - -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.runc] -runtime_type = 'io.containerd.runc.v2' - -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.runc.options] -BinaryName = '/usr/bin/runc' - -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.untrusted] -runtime_type = 'io.containerd.runc.v2' - -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.untrusted.options] -BinaryName = '/usr/bin/runc' - -[plugins.'io.containerd.grpc.v1.cri'.registry] -config_path = '/etc/containerd/certs.d' - -[plugins.'io.containerd.grpc.v1.cri'.registry.headers] -X-Meta-Source-Client = ['azure/aks'] - -[proxy_plugins] -[proxy_plugins.nydus-my-runtime] -type = 'snapshot' -address = '/run/containerd/containerd-nydus-grpc-my-runtime.sock' From 9438bdb662a96eab8076d378e1a6a9f0f3006e01 Mon Sep 17 00:00:00 2001 From: Paul Meyer <49727155+katexochen@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:02:41 +0100 Subject: [PATCH 13/19] Revert "node-installer: add kata config for AKS-PEER-SNP" This reverts commit 2ccaa8b03ae9b61b64022ff6a41eca3bf542c8d3. --- .../constants/configuration-peerpod.toml | 23 ------------------- nodeinstaller/internal/constants/constants.go | 19 --------------- 2 files changed, 42 deletions(-) delete mode 100644 nodeinstaller/internal/constants/configuration-peerpod.toml diff --git a/nodeinstaller/internal/constants/configuration-peerpod.toml b/nodeinstaller/internal/constants/configuration-peerpod.toml deleted file mode 100644 index 8f520fef1..000000000 --- a/nodeinstaller/internal/constants/configuration-peerpod.toml +++ /dev/null @@ -1,23 +0,0 @@ -# upstream source: https://github.com/kata-containers/kata-containers/blob/51bc71b8d96874cf4a555e1084ee07e948bff957/src/runtime/config/configuration-remote.toml.in -[hypervisor.remote] -remote_hypervisor_socket = "/run/peerpod/hypervisor.sock" -remote_hypervisor_timeout = 600 -enable_annotations = ["machine_type", "default_memory", "default_vcpus"] -firmware = "" -default_bridges = 1 -disable_selinux = false -disable_guest_selinux = true - -[agent.kata] - -[runtime] -internetworking_model = "none" -disable_guest_seccomp = true -disable_new_netns = true -sandbox_cgroup_only = false -static_sandbox_resource_mgmt = true -vfio_mode = "guest-kernel" -disable_guest_empty_dir = false -experimental = [] -create_container_timeout = 60 -dan_conf = "/run/edgeless/kata-containers/dans" diff --git a/nodeinstaller/internal/constants/constants.go b/nodeinstaller/internal/constants/constants.go index 9a263f45b..89628d918 100644 --- a/nodeinstaller/internal/constants/constants.go +++ b/nodeinstaller/internal/constants/constants.go @@ -8,7 +8,6 @@ import ( "fmt" "path/filepath" - "github.com/edgelesssys/contrast/internal/manifest" "github.com/edgelesssys/contrast/internal/platforms" "github.com/edgelesssys/contrast/nodeinstaller/internal/config" "github.com/pelletier/go-toml/v2" @@ -21,11 +20,6 @@ var ( //go:embed configuration-clh-snp.toml kataCLHSNPBaseConfig string - // kataPeerpodBaseConfig is the configuration file for the Kata runtime with peerpod. - // - //go:embed configuration-peerpod.toml - kataPeerpodBaseConfig string - // kataBareMetalQEMUTDXBaseConfig is the configuration file for the Kata runtime on bare-metal TDX // with QEMU. // @@ -101,15 +95,6 @@ func KataRuntimeConfig(baseDir string, platform platforms.Platform, qemuExtraKer // Replace the kernel params entirely (and don't append) since that's // also what we do when calculating the launch measurement. config.Hypervisor["qemu"]["kernel_params"] = kernelParams - case platforms.AKSPeerSNP: - if err := toml.Unmarshal([]byte(kataPeerpodBaseConfig), &config); err != nil { - return nil, fmt.Errorf("failed to unmarshal kata runtime configuration: %w", err) - } - runtimeHandlerName, err := manifest.RuntimeHandler(platform) - if err != nil { - return nil, fmt.Errorf("getting default runtime handler: %w", err) - } - config.Hypervisor["remote"]["remote_hypervisor_socket"] = filepath.Join("/run", "peerpod", fmt.Sprintf("hypervisor-%s.sock", runtimeHandlerName)) default: return nil, fmt.Errorf("unsupported platform: %s", platform) } @@ -153,10 +138,6 @@ func ContainerdRuntimeConfigFragment(baseDir, snapshotter string, platform platf cfg.Options = map[string]any{ "ConfigPath": filepath.Join(baseDir, "etc", "configuration-qemu-snp.toml"), } - case platforms.AKSPeerSNP: - cfg.Options = map[string]any{ - "ConfigPath": filepath.Join(baseDir, "etc", "configuration-peer-snp.toml"), - } default: return nil, fmt.Errorf("unsupported platform: %s", platform) } From 22ba580a5bdd674d26ef0da811aa740589dc52f1 Mon Sep 17 00:00:00 2001 From: Paul Meyer <49727155+katexochen@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:02:42 +0100 Subject: [PATCH 14/19] Revert "genpolicy: use kata genpolicy for AKS-PEER-SNP" This reverts commit ed1355ed197c92349a45ac7454fe436d16e40b8a. --- cli/genpolicy/config.go | 2 +- cli/main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/genpolicy/config.go b/cli/genpolicy/config.go index b61a717f2..19a5c0761 100644 --- a/cli/genpolicy/config.go +++ b/cli/genpolicy/config.go @@ -43,7 +43,7 @@ func NewConfig(platform platforms.Platform) *Config { Settings: aksSettings, Bin: aksGenpolicyBin, } - case platforms.AKSPeerSNP, platforms.K3sQEMUSNP, platforms.K3sQEMUTDX, platforms.RKE2QEMUTDX: + case platforms.K3sQEMUSNP, platforms.K3sQEMUTDX, platforms.RKE2QEMUTDX: return &Config{ Rules: kataRules, Settings: kataSettings, diff --git a/cli/main.go b/cli/main.go index 9bc6e1589..54e3889e1 100644 --- a/cli/main.go +++ b/cli/main.go @@ -105,7 +105,7 @@ func buildVersionString() (string, error) { switch platform { case platforms.AKSCloudHypervisorSNP: fmt.Fprintf(versionsWriter, "\tgenpolicy version:\t%s\n", constants.MicrosoftGenpolicyVersion) - case platforms.AKSPeerSNP, platforms.K3sQEMUSNP, platforms.K3sQEMUTDX, platforms.RKE2QEMUTDX: + case platforms.K3sQEMUSNP, platforms.K3sQEMUTDX, platforms.RKE2QEMUTDX: fmt.Fprintf(versionsWriter, "\tgenpolicy version:\t%s\n", constants.KataGenpolicyVersion) } } From da32110aa152df06c6e656ed1d1a22e002d07baa Mon Sep 17 00:00:00 2001 From: Paul Meyer <49727155+katexochen@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:02:42 +0100 Subject: [PATCH 15/19] Revert "platforms: add AKS-PEER-SNP" This reverts commit 4b43d407011f302403570badf764af0339ba9d18. --- internal/platforms/platforms.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/internal/platforms/platforms.go b/internal/platforms/platforms.go index eea1e01b4..92966870c 100644 --- a/internal/platforms/platforms.go +++ b/internal/platforms/platforms.go @@ -18,8 +18,6 @@ const ( Unknown Platform = iota // AKSCloudHypervisorSNP represents a deployment with Cloud-Hypervisor on SEV-SNP AKS. AKSCloudHypervisorSNP - // AKSPeerSNP represents a deployment with peer-pods on SEV-SNP AKS. - AKSPeerSNP // K3sQEMUTDX represents a deployment with QEMU on bare-metal TDX K3s. K3sQEMUTDX // K3sQEMUSNP represents a deployment with QEMU on bare-metal SNP K3s. @@ -30,7 +28,7 @@ const ( // All returns a list of all available platforms. func All() []Platform { - return []Platform{AKSCloudHypervisorSNP, AKSPeerSNP, K3sQEMUTDX, K3sQEMUSNP, RKE2QEMUTDX} + return []Platform{AKSCloudHypervisorSNP, K3sQEMUTDX, K3sQEMUSNP, RKE2QEMUTDX} } // AllStrings returns a list of all available platforms as strings. @@ -47,8 +45,6 @@ func (p Platform) String() string { switch p { case AKSCloudHypervisorSNP: return "AKS-CLH-SNP" - case AKSPeerSNP: - return "AKS-PEER-SNP" case K3sQEMUTDX: return "K3s-QEMU-TDX" case K3sQEMUSNP: @@ -65,8 +61,6 @@ func FromString(s string) (Platform, error) { switch strings.ToLower(s) { case "aks-clh-snp": return AKSCloudHypervisorSNP, nil - case "aks-peer-snp": - return AKSPeerSNP, nil case "k3s-qemu-tdx": return K3sQEMUTDX, nil case "k3s-qemu-snp": From ff4bcfe13f61a056d69a406f604498f798886f04 Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Wed, 13 Nov 2024 14:20:42 +0100 Subject: [PATCH 16/19] scripts: create ~/.kube if it does not exist --- packages/scripts.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/scripts.nix b/packages/scripts.nix index 02d762ad9..9050e1fe1 100644 --- a/packages/scripts.nix +++ b/packages/scripts.nix @@ -403,7 +403,9 @@ newContext=$(yq -r '.contexts.[0].name' "$1") declare -x newContext yq -i '.current-context = env(newContext)' "$mergedConfig" - mv "$mergedConfig" "''${KUBECONFIG_BAK%%:*}" + targetFile="''${KUBECONFIG_BAK%%:*}" + mkdir -p "$(dirname "$targetFile")" + mv "$mergedConfig" "$targetFile" ''; }; From 10764aad35792c87441bad3836e1e79071585451 Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Fri, 15 Nov 2024 08:55:03 +0100 Subject: [PATCH 17/19] scripts: support dashes in resource group for upload-image --- packages/scripts.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/scripts.nix b/packages/scripts.nix index 9050e1fe1..92d9934ee 100644 --- a/packages/scripts.nix +++ b/packages/scripts.nix @@ -69,7 +69,7 @@ subscriptionID = "''${subscriptionId}" location = "''${location}" resourceGroup = "''${resourceGroup}" - sharedImageGallery = "''${resourceGroup}_contrast" + sharedImageGallery = "''${resourceGroup//-/_}_contrast" sharingProfile = "private" EOF From ade385249fd98a4b7e6b96b7b8d09ce78ab38e50 Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Fri, 15 Nov 2024 15:54:38 +0100 Subject: [PATCH 18/19] infra: separate IAM from AKS and image Co-authored-by: Paul Meyer <49727155+katexochen@users.noreply.github.com> --- .gitignore | 1 - infra/azure-peerpods-iam/.terraform.lock.hcl | 62 ++++++++++++++ infra/azure-peerpods-iam/main.tf | 75 +++++++++++++++++ infra/azure-peerpods-iam/outs.tf | 8 ++ infra/azure-peerpods-iam/vars.tf | 11 +++ infra/azure-peerpods/.terraform.lock.hcl | 30 ------- infra/azure-peerpods/main.tf | 89 ++------------------ infra/azure-peerpods/vars.tf | 17 +++- justfile | 69 ++++++++++++--- packages/scripts.nix | 5 -- 10 files changed, 235 insertions(+), 132 deletions(-) create mode 100644 infra/azure-peerpods-iam/.terraform.lock.hcl create mode 100644 infra/azure-peerpods-iam/main.tf create mode 100644 infra/azure-peerpods-iam/outs.tf create mode 100644 infra/azure-peerpods-iam/vars.tf diff --git a/.gitignore b/.gitignore index 3f764fc51..9e0eb180e 100644 --- a/.gitignore +++ b/.gitignore @@ -22,5 +22,4 @@ id_rsa* kube.conf out.env infra/**/kustomization.yaml -infra/**/workload-identity.yaml uplosi.conf* diff --git a/infra/azure-peerpods-iam/.terraform.lock.hcl b/infra/azure-peerpods-iam/.terraform.lock.hcl new file mode 100644 index 000000000..cf2b56c5a --- /dev/null +++ b/infra/azure-peerpods-iam/.terraform.lock.hcl @@ -0,0 +1,62 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azuread" { + version = "3.0.2" + constraints = "3.0.2" + hashes = [ + "h1:sYCyzbPpSYu2XDah8XqBUITQAfB0x4j4Twh6lw2C4CA=", + "zh:16e724b80a9004c7978c30f69a73c98ff63eb8a03937dd44c2a8f0ea0438b7a3", + "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", + "zh:2bbbf13713ca4767267b889471c9fc14a56a8fdf5d1013da3ca78667e3caec64", + "zh:409ccb05431d643a079da082d89db2d95d6afed4769997ac537c8b7de3bff867", + "zh:53e4bca0f5d015380f7f524f36344afe6211ccaf614bfc69af73ca64a9f47d6c", + "zh:5780be2c1981d090604d7fa4cef675462f17f40e7f3dc501a031488e87a35b8f", + "zh:850e61a1b3e64c752c418526ccf48653514c861b36f5feb631619f906f7e99a0", + "zh:8c3565bfcea006a734149cc080452a9daf7d2a9d5362eb7e0a088b6c0d7f0f03", + "zh:908b9e6ad49d5d21173ecefc7924902047611be93bbf8e7d021aa9563358396f", + "zh:a2a79765c029bc58966eff61cb6e9b0ee14d2ac52b0a22fc7dfa35c9a49af669", + "zh:c7f56cbe8743e9ba81fce871bc97d9c07abe86770d9ee7ffefbf3882a61ba89a", + "zh:d4dba80e33421b30d81c62611fb7fc62ad39afecc6484436e635913cd8553e67", + ] +} + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "4.5.0" + constraints = "4.5.0" + hashes = [ + "h1:bAEb9HTc1Yl0ULs+WQAI6jAoKWv4I2LUGpoESf/iCyc=", + "zh:27ac12977bdb7b82217a3fe35d3206e1e4261465d738aff93244ec90f2bd431a", + "zh:36a619af3767a92ee892c5de24604eeb9f23a5a01bb8455115a5eb4bd656f234", + "zh:45a374637b794427c5e07d23c6312d92d58bed3594789322c109d333ea1865e5", + "zh:538e501d313cfc0b61f3b2e5be9ae7755df3d3d9a3e4f14e0ea6a943d5102109", + "zh:64d8e4b94a1324292fe318bf27c6149aa345eabab8b89d9d78ce447ce5600e65", + "zh:7b3fcc0a724c5e00e6ce0e7da22010b6ae4bd2622544ef4d31fd4100f85985d7", + "zh:84876a614b010ae5dbef1b1edd9a22447cf57b9300b9eaf4321d587bfebf82dc", + "zh:850e3900fb2b55ad85b6def8b580fb851778bb470be5354cb0a0244d03acd5a4", + "zh:b6355d1eb7d165b246ad9c8f7c0ce7ccd5bbc58a01bd853c7ca896c71f4cd295", + "zh:bd4f1558f24af356d372937b810801555471eafbbc0552471bb6760f8ddd6b7e", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f78eaaf507ab56041112b765f6ca1740221773f3b32710bb8d087f29a686f30f", + ] +} + +provider "registry.terraform.io/hashicorp/local" { + version = "2.5.2" + constraints = "2.5.2" + hashes = [ + "h1:JlMZD6nYqJ8sSrFfEAH0Vk/SL8WLZRmFaMUF9PJK5wM=", + "zh:136299545178ce281c56f36965bf91c35407c11897f7082b3b983d86cb79b511", + "zh:3b4486858aa9cb8163378722b642c57c529b6c64bfbfc9461d940a84cd66ebea", + "zh:4855ee628ead847741aa4f4fc9bed50cfdbf197f2912775dd9fe7bc43fa077c0", + "zh:4b8cd2583d1edcac4011caafe8afb7a95e8110a607a1d5fb87d921178074a69b", + "zh:52084ddaff8c8cd3f9e7bcb7ce4dc1eab00602912c96da43c29b4762dc376038", + "zh:71562d330d3f92d79b2952ffdda0dad167e952e46200c767dd30c6af8d7c0ed3", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:805f81ade06ff68fa8b908d31892eaed5c180ae031c77ad35f82cb7a74b97cf4", + "zh:8b6b3ebeaaa8e38dd04e56996abe80db9be6f4c1df75ac3cccc77642899bd464", + "zh:ad07750576b99248037b897de71113cc19b1a8d0bc235eb99173cc83d0de3b1b", + "zh:b9f1c3bfadb74068f5c205292badb0661e17ac05eb23bfe8bd809691e4583d0e", + "zh:cc4cbcd67414fefb111c1bf7ab0bc4beb8c0b553d01719ad17de9a047adff4d1", + ] +} diff --git a/infra/azure-peerpods-iam/main.tf b/infra/azure-peerpods-iam/main.tf new file mode 100644 index 000000000..cab9d062b --- /dev/null +++ b/infra/azure-peerpods-iam/main.tf @@ -0,0 +1,75 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "4.5.0" + } + azuread = { + source = "hashicorp/azuread" + version = "3.0.2" + } + local = { + source = "hashicorp/local" + version = "2.5.2" + } + } +} + +provider "azurerm" { + subscription_id = var.subscription_id + features { + resource_group { + prevent_deletion_if_contains_resources = false + } + } +} + +data "azurerm_subscription" "current" {} + +data "azuread_client_config" "current" {} + +provider "azuread" { + tenant_id = data.azurerm_subscription.current.tenant_id +} + +locals { + name = var.resource_group +} + +resource "azurerm_resource_group" "rg" { + name = var.resource_group + location = var.location +} + +resource "azuread_application" "app" { + display_name = "${local.name}-app" + owners = [data.azuread_client_config.current.object_id] +} + +resource "azuread_service_principal" "sp" { + client_id = azuread_application.app.client_id + app_role_assignment_required = false + owners = [data.azuread_client_config.current.object_id] +} + +resource "azurerm_role_assignment" "ra_vm_contributor" { + scope = azurerm_resource_group.rg.id + role_definition_name = "Virtual Machine Contributor" + principal_id = azuread_service_principal.sp.object_id +} + +resource "azurerm_role_assignment" "ra_reader" { + scope = azurerm_resource_group.rg.id + role_definition_name = "Reader" + principal_id = azuread_service_principal.sp.object_id +} + +resource "azurerm_role_assignment" "ra_network_contributor" { + scope = azurerm_resource_group.rg.id + role_definition_name = "Network Contributor" + principal_id = azuread_service_principal.sp.object_id +} + +resource "azuread_application_password" "pw" { + application_id = azuread_application.app.id +} diff --git a/infra/azure-peerpods-iam/outs.tf b/infra/azure-peerpods-iam/outs.tf new file mode 100644 index 000000000..64b692543 --- /dev/null +++ b/infra/azure-peerpods-iam/outs.tf @@ -0,0 +1,8 @@ +output "client_secret_env" { + value = < infra/azure-peerpods-iam/just.auto.tfvars + echo "location = \"${azure_location}\"" >> infra/azure-peerpods-iam/just.auto.tfvars + echo "subscription_id = \"${azure_subscription_id}\"" >> infra/azure-peerpods-iam/just.auto.tfvars + nix run -L .#terraform -- -chdir=infra/azure-peerpods-iam init + nix run -L .#terraform -- -chdir=infra/azure-peerpods-iam apply --auto-approve + nix run -L .#terraform -- -chdir=infra/azure-peerpods-iam output -raw client_secret_env > infra/azure-peerpods/iam.auto.tfvars + echo "resource_group = \"${azure_resource_group}_caa_cluster\"" >> infra/azure-peerpods/iam.auto.tfvars + + # TODO(burgerdev): this should be done in a generic upload target, together with OCI images + just upload-image + ;; + *) + echo "Unsupported platform: {{ platform }}" + exit 1 + ;; + esac + # Create a CoCo-enabled AKS cluster. create platform=default_platform: #!/usr/bin/env bash @@ -193,15 +219,11 @@ create platform=default_platform: : ;; "AKS-PEER-SNP") - just upload-image - # Populate Terraform variables. - echo "name_prefix = \"$azure_resource_group\"" > infra/azure-peerpods/just.auto.tfvars - echo "image_resource_group_name = \"$azure_resource_group\"" >> infra/azure-peerpods/just.auto.tfvars - echo "subscription_id = \"$azure_subscription_id\"" >> infra/azure-peerpods/just.auto.tfvars + echo "subscription_id = \"$azure_subscription_id\"" > infra/azure-peerpods/just.auto.tfvars nix run -L .#terraform -- -chdir=infra/azure-peerpods init - nix run -L .#terraform -- -chdir=infra/azure-peerpods apply + nix run -L .#terraform -- -chdir=infra/azure-peerpods apply --auto-approve ;; *) echo "Unsupported platform: {{ platform }}" @@ -330,12 +352,35 @@ destroy platform=default_platform: : ;; "AKS-PEER-SNP") - nix run -L .#terraform -- -chdir=infra/azure-peerpods destroy + nix run -L .#terraform -- -chdir=infra/azure-peerpods destroy --auto-approve # Clean-up cached image ids. rm -f ${CONTRAST_CACHE_DIR}/image-upload/*.image-id - az group delete --name "${azure_resource_group}_caa_cluster" --yes + ;; + *) + echo "Unsupported platform: {{ platform }}" + exit 1 + ;; + esac + +# Destroy foundational dependencies +destroy-post platform=default_platform: + #!/usr/bin/env bash + set -euo pipefail + case {{ platform }} in + "AKS-CLH-SNP") + # TODO(burgerdev): this should destroy the resource group for consistency. + : + ;; + "K3s-QEMU-SNP"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX") + : + ;; + "AKS-PEER-SNP") + nix run -L .#terraform -- -chdir=infra/azure-peerpods-iam destroy --auto-approve + + # We just destroyed the resource group, so these IDs are invalid. + rm -f ${CONTRAST_CACHE_DIR}/image-upload/*.image-id ;; *) echo "Unsupported platform: {{ platform }}" diff --git a/packages/scripts.nix b/packages/scripts.nix index 92d9934ee..22fabeee4 100644 --- a/packages/scripts.nix +++ b/packages/scripts.nix @@ -466,10 +466,6 @@ kustomizationFile="''${i#*=}" shift ;; - --workload-identity=*) - workloadIdentityFile="''${i#*=}" - shift - ;; --pub-key=*) pubKeyFile="''${i#*=}" shift @@ -485,7 +481,6 @@ cp -r ${pkgs.cloud-api-adaptor.src}/src/cloud-api-adaptor/install/* "$tmpdir" chmod -R +w "$tmpdir" cp "$kustomizationFile" "$tmpdir/overlays/azure/kustomization.yaml" - cp "$workloadIdentityFile" "$tmpdir/overlays/azure/workload-identity.yaml" cp "$pubKeyFile" "$tmpdir/overlays/azure/id_rsa.pub" kubectl apply -k "github.com/confidential-containers/operator/config/release?ref=v${pkgs.cloud-api-adaptor.version}" From dd18dc4ab162706132c81ae335f0a232dd29ac86 Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Fri, 15 Nov 2024 15:55:22 +0100 Subject: [PATCH 19/19] e2e: smoke test for peer pods Co-authored-by: Paul Meyer <49727155+katexochen@users.noreply.github.com> --- .github/workflows/e2e_peerpods.yml | 50 ++++++++++++++++++++++++++ packages/scripts.nix | 10 ++++++ packages/test-peerpods.sh | 58 ++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 .github/workflows/e2e_peerpods.yml create mode 100644 packages/test-peerpods.sh diff --git a/.github/workflows/e2e_peerpods.yml b/.github/workflows/e2e_peerpods.yml new file mode 100644 index 000000000..9b0c5cf53 --- /dev/null +++ b/.github/workflows/e2e_peerpods.yml @@ -0,0 +1,50 @@ +name: e2e peer-pods + +on: + workflow_dispatch: + inputs: + image-id: + description: "ID of the guest VM image to test (default: build a fresh image)" + required: false + pull_request: + paths: + - .github/workflows/e2e_peerpods.yml + - packages/test-peerpods.sh + - packages/by-name/cloud-api-adaptor/** + - packages/by-name/kata/** + - packages/by-name/image-podvm/** + - packages/nixos + +jobs: + test: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: ./.github/actions/setup_nix + with: + githubToken: ${{ secrets.GITHUB_TOKEN }} + cachixToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + - name: Login to Azure + uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0 + with: + creds: ${{ secrets.CONTRAST_CI_INFRA_AZURE }} + - name: Test peer-pods + env: + azure_subscription_id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + azure_image_id: ${{ inputs.image-id }} + azure_resource_group: contrast-ci + azure_location: germanywestcentral + CONTRAST_CACHE_DIR: "./workspace.cache" + run: | + ssh-keygen -t rsa -f ./infra/azure-peerpods/id_rsa -N "" + cat >infra/azure-peerpods/iam.auto.tfvars <infra/azure-peerpods/image_id.auto.tfvars +fi + +cat >infra/azure-peerpods/e2e.auto.tfvars <