From 31478f71fec88d783774c038dea844b86f59a436 Mon Sep 17 00:00:00 2001
From: Moritz Sanft <58110325+msanft@users.noreply.github.com>
Date: Mon, 28 Oct 2024 13:17:28 +0100
Subject: [PATCH] packages/nvidia-ctk-with-config: init

This adds a preconfigured `nvidia-ctk` package for use with peer pods GPU containers.
---
 packages/by-name/mkNixosConfig/package.nix    |  1 +
 .../nvidia-ctk-with-config/config.toml        | 40 +++++++++++++++++++
 .../nvidia-ctk-with-config/package.nix        | 21 ++++++++++
 3 files changed, 62 insertions(+)
 create mode 100644 packages/by-name/nvidia-ctk-with-config/config.toml
 create mode 100644 packages/by-name/nvidia-ctk-with-config/package.nix

diff --git a/packages/by-name/mkNixosConfig/package.nix b/packages/by-name/mkNixosConfig/package.nix
index 1e00b77a2c..c4187f7b9a 100644
--- a/packages/by-name/mkNixosConfig/package.nix
+++ b/packages/by-name/mkNixosConfig/package.nix
@@ -41,6 +41,7 @@ lib.makeOverridable (
             cloud-api-adaptor
             kernel-podvm-azure
             pause-bundle
+            nvidia-ctk-with-config
             ;
           inherit (outerPkgs.kata) kata-agent;
         })
diff --git a/packages/by-name/nvidia-ctk-with-config/config.toml b/packages/by-name/nvidia-ctk-with-config/config.toml
new file mode 100644
index 0000000000..1ae191dd61
--- /dev/null
+++ b/packages/by-name/nvidia-ctk-with-config/config.toml
@@ -0,0 +1,40 @@
+#accept-nvidia-visible-devices-as-volume-mounts = false
+#accept-nvidia-visible-devices-envvar-when-unprivileged = true
+disable-require = true
+supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
+#swarm-resource = "DOCKER_RESOURCE_GPU"
+
+[nvidia-container-cli]
+no-pivot = true
+debug = "/var/log/nvidia-kata-container/nvidia-container-toolkit.log"
+environment = []
+ldcache = "/tmp/ld.so.cache"
+ldconfig = "@@glibcbin@/bin/ldconfig"
+load-kmods = true
+no-cgroups = true
+path = "@nvidia-container-cli@"
+#root = "/run/nvidia/driver"
+#user = "root:video"
+
+[nvidia-container-runtime]
+debug = "/var/log/nvidia-kata-container/nvidia-container-runtime.log"
+log-level = "debug"
+mode = "cdi"
+runtimes = ["docker-runc", "runc", "crun"]
+
+[nvidia-container-runtime.modes]
+
+[nvidia-container-runtime.modes.cdi]
+annotation-prefixes = ["cdi.k8s.io/"]
+default-kind = "nvidia.com/gpu"
+spec-dirs = ["/var/run/cdi"]
+
+[nvidia-container-runtime.modes.csv]
+mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
+
+[nvidia-container-runtime-hook]
+path = "@nvidia-container-runtime-hook@"
+skip-mode-detection = true
+
+[nvidia-ctk]
+path = "@nvidia-ctk@"
diff --git a/packages/by-name/nvidia-ctk-with-config/package.nix b/packages/by-name/nvidia-ctk-with-config/package.nix
new file mode 100644
index 0000000000..bdf57a1910
--- /dev/null
+++ b/packages/by-name/nvidia-ctk-with-config/package.nix
@@ -0,0 +1,21 @@
+# Copyright 2024 Edgeless Systems GmbH
+# SPDX-License-Identifier: AGPL-3.0-only
+
+# This builds an nvidia-container-toolkit package with a custom config required
+# for use in peer pods GPU containers.
+
+{
+  nvidia-container-toolkit,
+  libnvidia-container,
+  replaceVars,
+  glibc,
+  lib,
+}:
+nvidia-container-toolkit.override {
+  configTemplatePath = replaceVars ./config.toml {
+    "nvidia-container-cli" = "${lib.getExe' libnvidia-container "nvidia-container-cli"}";
+    "nvidia-container-runtime-hook" = "${lib.getExe' nvidia-container-toolkit "nvidia-container-runtime-hook"}";
+    "nvidia-ctk" = "${lib.getExe' nvidia-container-toolkit "nvidia-ctk"}";
+    "glibcbin" = "${lib.getBin glibc}";
+  };
+}