From 50a87ce87c2a32c0e563039ccc370507e06a32ed Mon Sep 17 00:00:00 2001
From: Markus Rudy <mr@edgeless.systems>
Date: Wed, 27 Nov 2024 12:17:02 +0100
Subject: [PATCH] caa: use our own image

Adjust the upstream cloud-api-adaptor DaemonSet to the idiosyncracies of
our Nix-built container image, and vice-versa.

* Set the HOME env var so that the app can find the SSH key.
* Add a mount point for /var/run/netns because our image does not
  symlink /var/run to /run.
* Add a stable symlink to the real entrypoint at the expected location.

Furthermore, add the required iptables binary as a runtime dependency to
cloud-api-adaptor.
---
 .gitignore                                    |  1 +
 infra/azure-peerpods/main.tf                  | 27 +++++++++++++++++++
 infra/azure-peerpods/vars.tf                  |  4 +++
 justfile                                      |  7 +++--
 .../by-name/cloud-api-adaptor/package.nix     |  1 +
 packages/containers.nix                       |  4 +++
 packages/scripts.nix                          | 17 +++++-------
 7 files changed, 48 insertions(+), 13 deletions(-)

diff --git a/.gitignore b/.gitignore
index 9e0eb180e..32c87f968 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,4 +22,5 @@ id_rsa*
 kube.conf
 out.env
 infra/**/kustomization.yaml
+infra/**/patch_ds.yaml
 uplosi.conf*
diff --git a/infra/azure-peerpods/main.tf b/infra/azure-peerpods/main.tf
index 83e980d39..127673fad 100644
--- a/infra/azure-peerpods/main.tf
+++ b/infra/azure-peerpods/main.tf
@@ -137,5 +137,32 @@ secretGenerator:
   namespace: confidential-containers-system
   files:
   - id_rsa.pub
+patches:
+- path: patch_ds.yaml
+EOF
+}
+
+resource "local_file" "patch_ds" {
+  filename        = "./patch_ds.yaml"
+  file_permission = "0777"
+  content         = <<EOF
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: cloud-api-adaptor-daemonset
+  namespace: confidential-containers-system
+spec:
+  template:
+    spec:
+      containers:
+      - name: cloud-api-adaptor-con
+        image: ${var.caa_image}
+        env:
+        - name: HOME
+          value: /root
+        volumeMounts:
+        - name: netns
+          mountPropagation: HostToContainer
+          mountPath: /var/run/netns
 EOF
 }
diff --git a/infra/azure-peerpods/vars.tf b/infra/azure-peerpods/vars.tf
index 570a8c76c..5143a8194 100644
--- a/infra/azure-peerpods/vars.tf
+++ b/infra/azure-peerpods/vars.tf
@@ -27,6 +27,10 @@ variable "image_id" {
   type = string
 }
 
+variable "caa_image" {
+  type = string
+}
+
 variable "cluster_type" {
   type    = string
   default = "Free"
diff --git a/justfile b/justfile
index 58e10e903..c36425ffd 100644
--- a/justfile
+++ b/justfile
@@ -53,8 +53,9 @@ node-installer platform=default_platform:
         ;;
         "AKS-PEER-SNP")
             nix run -L .#scripts.deploy-caa -- \
-                --kustomization=./infra/azure-peerpods/kustomization.yaml \
-                --pub-key=./infra/azure-peerpods/id_rsa.pub
+                --copy=./infra/azure-peerpods/kustomization.yaml \
+                --copy=./infra/azure-peerpods/patch_ds.yaml \
+                --copy=./infra/azure-peerpods/id_rsa.pub
         ;;
         *)
             echo "Unsupported platform: {{ platform }}"
@@ -220,7 +221,9 @@ create platform=default_platform:
         ;;
         "AKS-PEER-SNP")
             # Populate Terraform variables.
+            image=$(nix run -L .#containers.push-cloud-api-adaptor -- "$container_registry/contrast/cloud-api-adaptor")
             echo "subscription_id = \"$azure_subscription_id\"" > infra/azure-peerpods/just.auto.tfvars
+            echo "caa_image = \"$image\"" >> infra/azure-peerpods/just.auto.tfvars
 
             nix run -L .#terraform -- -chdir=infra/azure-peerpods init
             nix run -L .#terraform -- -chdir=infra/azure-peerpods apply --auto-approve
diff --git a/packages/by-name/cloud-api-adaptor/package.nix b/packages/by-name/cloud-api-adaptor/package.nix
index 1e1e91520..4f52f67cc 100644
--- a/packages/by-name/cloud-api-adaptor/package.nix
+++ b/packages/by-name/cloud-api-adaptor/package.nix
@@ -77,6 +77,7 @@ buildGoModule rec {
 
   postInstall = ''
     wrapProgram $out/bin/agent-protocol-forwarder --prefix PATH : ${lib.makeBinPath [ iptables ]}
+    wrapProgram $out/bin/cloud-api-adaptor --prefix PATH : ${lib.makeBinPath [ iptables ]}
   '';
 
   passthru = {
diff --git a/packages/containers.nix b/packages/containers.nix
index 78e277ca5..70f1f5b7f 100644
--- a/packages/containers.nix
+++ b/packages/containers.nix
@@ -173,6 +173,10 @@ let
       copyToRoot = with pkgs; [
         cacert
       ];
+      runAsRoot = ''
+        mkdir -p /usr/local/bin
+        ln -s "${lib.getExe pkgs.cloud-api-adaptor.entrypoint}" /usr/local/bin/entrypoint.sh
+      '';
       config = {
         Cmd = [ "${lib.getExe pkgs.cloud-api-adaptor.entrypoint}" ];
       };
diff --git a/packages/scripts.nix b/packages/scripts.nix
index 3f977ec2a..b8748cb69 100644
--- a/packages/scripts.nix
+++ b/packages/scripts.nix
@@ -470,14 +470,14 @@
     text = ''
       set -euo pipefail
 
+      tmpdir=$(mktemp -d)
+      cp -r ${pkgs.cloud-api-adaptor.src}/src/cloud-api-adaptor/install/* "$tmpdir"
+      chmod -R +w "$tmpdir"
+
       for i in "$@"; do
         case $i in
-        --kustomization=*)
-          kustomizationFile="''${i#*=}"
-          shift
-          ;;
-        --pub-key=*)
-          pubKeyFile="''${i#*=}"
+        --copy=*)
+          cp "''${i#*=}" "$tmpdir/overlays/azure/"
           shift
           ;;
         *)
@@ -487,11 +487,6 @@
         esac
       done
 
-      tmpdir=$(mktemp -d)
-      cp -r ${pkgs.cloud-api-adaptor.src}/src/cloud-api-adaptor/install/* "$tmpdir"
-      chmod -R +w "$tmpdir"
-      cp "$kustomizationFile" "$tmpdir/overlays/azure/kustomization.yaml"
-      cp "$pubKeyFile" "$tmpdir/overlays/azure/id_rsa.pub"
 
       kubectl apply -k "github.com/confidential-containers/operator/config/release?ref=v${pkgs.cloud-api-adaptor.version}"
       kubectl apply -k "github.com/confidential-containers/operator/config/samples/ccruntime/peer-pods?ref=v${pkgs.cloud-api-adaptor.version}"