From c6f2050fe0f09284026522fe22cc0e538e45fc33 Mon Sep 17 00:00:00 2001
From: Malte Poll <1780588+malt3@users.noreply.github.com>
Date: Fri, 15 Mar 2024 16:04:46 +0100
Subject: [PATCH] nix: create efficient oci images with reusable layers

This set of functions generates layers that can be combined into OCI image layout directories.
Those can directly be pushed to a registry.
Strong focus was placed on making the layers reusable in arbitrary ways.
This allows for efficient distribution of binary artifacts via container image layers.
---
 .../config/vocabularies/edgeless/accept.txt   |  3 +
 packages/by-name/ociImageConfig/README.md     |  1 +
 packages/by-name/ociImageConfig/package.nix   | 36 +++++++++++
 packages/by-name/ociImageLayout/README.md     | 55 ++++++++++++++++
 packages/by-name/ociImageLayout/package.nix   | 40 ++++++++++++
 packages/by-name/ociImageManifest/README.md   |  1 +
 packages/by-name/ociImageManifest/package.nix | 46 ++++++++++++++
 packages/by-name/ociLayerTar/README.md        |  1 +
 packages/by-name/ociLayerTar/package.nix      | 63 +++++++++++++++++++
 9 files changed, 246 insertions(+)
 create mode 120000 packages/by-name/ociImageConfig/README.md
 create mode 100644 packages/by-name/ociImageConfig/package.nix
 create mode 100644 packages/by-name/ociImageLayout/README.md
 create mode 100644 packages/by-name/ociImageLayout/package.nix
 create mode 120000 packages/by-name/ociImageManifest/README.md
 create mode 100644 packages/by-name/ociImageManifest/package.nix
 create mode 120000 packages/by-name/ociLayerTar/README.md
 create mode 100644 packages/by-name/ociLayerTar/package.nix

diff --git a/docs/styles/config/vocabularies/edgeless/accept.txt b/docs/styles/config/vocabularies/edgeless/accept.txt
index 9610913be8..e1d431c9a3 100644
--- a/docs/styles/config/vocabularies/edgeless/accept.txt
+++ b/docs/styles/config/vocabularies/edgeless/accept.txt
@@ -13,7 +13,9 @@ backport
 Bazel
 bootloader
 Bootstrapper
+cachable
 cachix
+changeset
 cloud
 cmdline
 config
@@ -66,6 +68,7 @@ Nginx
 paravisor
 PCR
 plaintext
+podman
 protobuf
 proxied
 QEMU
diff --git a/packages/by-name/ociImageConfig/README.md b/packages/by-name/ociImageConfig/README.md
new file mode 120000
index 0000000000..4126071467
--- /dev/null
+++ b/packages/by-name/ociImageConfig/README.md
@@ -0,0 +1 @@
+../ociImageLayout/README.md
\ No newline at end of file
diff --git a/packages/by-name/ociImageConfig/package.nix b/packages/by-name/ociImageConfig/package.nix
new file mode 100644
index 0000000000..deb0e70715
--- /dev/null
+++ b/packages/by-name/ociImageConfig/package.nix
@@ -0,0 +1,36 @@
+# application/vnd.oci.image.config.v1+json
+{ lib, runCommand, writers, nix }:
+{
+  # layers is a list of ociLayerTar
+  layers ? [ ]
+  # extraConfig is a set of extra configuration options
+, extraConfig ? { }
+}:
+let
+  diffIDs = lib.lists.map (layer: builtins.readFile (layer + "/DiffID")) layers;
+  config = {
+    architecture = "amd64";
+    os = "linux";
+  } // extraConfig // {
+    rootfs = { type = "layers"; diff_ids = diffIDs; };
+  };
+  configJSON = writers.writeJSON "image-config.json" config;
+in
+runCommand "oci-image-config"
+{
+  buildInputs = [ nix ];
+  platformJSON = builtins.toJSON { inherit (config) architecture; inherit (config) os; };
+  inherit configJSON;
+} ''
+  # write the config to a file under blobs/sha256
+  mkdir -p $out/blobs/sha256
+  sha256=$(nix-hash --type sha256 --flat $configJSON)
+  cp $configJSON "$out/blobs/sha256/$sha256"
+
+  # create a symlink to the image config
+  ln -s "$out/blobs/sha256/$sha256" "$out/image-config.json"
+  # write the platform.json
+  echo "$platformJSON" > "$out/platform.json"
+  # write the media descriptor
+  echo -n "{\"mediaType\": \"application/vnd.oci.image.config.v1+json\", \"size\": $(stat -c %s $configJSON), \"digest\": \"sha256:$sha256\"}" > $out/media-descriptor.json
+''
diff --git a/packages/by-name/ociImageLayout/README.md b/packages/by-name/ociImageLayout/README.md
new file mode 100644
index 0000000000..8b6582b4d7
--- /dev/null
+++ b/packages/by-name/ociImageLayout/README.md
@@ -0,0 +1,55 @@
+# OCI image tools
+
+This is a set of nix functions for creating reproducible and cachable multi-layer OCI images.
+
+It uses the following functions:
+
+- `ociImageLayout`: Top level function for creating an [OCI image directory layout](https://github.com/opencontainers/image-spec/blob/v1.1.0/image-layout.md). This can be used directly (by podman) or uploaded to a registry (`crane push path/to/directory registry/image/name:tag`).
+- `ociImageManifest`: An [OCI image manifest](https://github.com/opencontainers/image-spec/blob/v1.1.0/manifest.md#image-manifest) that can be added to the top-level layout. A manifest contains a configuration and layers.
+- `ociImageConfig`: An [OCI image configuration](https://github.com/opencontainers/image-spec/blob/v1.1.0/config.md) that can be included in a manifest. The configuration describes the image including layers, entrypoint, arguments, architecture, os and more.
+- `ociLayerTar`: An [OCI image layer filesystem changeset (layer tar)](https://github.com/opencontainers/image-spec/blob/v1.1.0/layer.md). Contains an individual container image layer. Can be freely remixed with other layers. Takes a list of store paths and their target destinations in the image.
+
+## Example
+
+The following example creates an image containing two layers:
+
+- one layer for nginx
+- one layer with `bash`, `jq`, a script and configuration
+
+```nix
+ociImageLayout {
+  manifests = [
+    ociImageManifest
+    {
+      layers = [
+        ociLayerTar {
+            files = [ { source = nginx; } ];
+        }
+        ociLayerTar {
+            files = [
+             { source = bash; destination = "/bin/bash"; }
+             { source = jq; destination = "/bin/jq"; }
+             { source = writeShellScript "entrypoint.sh" '' jq $CONFIG_PATH ; nginx -g 'daemon off;' ''; destination = "/entrypoint.sh"; }
+             { source = writers.writeJSON "conf.json" { a = 1; b = 2; }; destination = "/etc/configuration.json"; }
+            ];
+        }
+      ];
+      extraConfig = {
+        "config" = {
+          "Env" = [
+            "PATH=/bin:/usr/bin"
+            "CONFIG_PATH=/config"
+          ];
+          "Entrypoint" = [ "/entrypoint.sh" ];
+        };
+      };
+      extraManifest = {
+        "annotations" = {
+          "org.opencontainers.image.title" = "example-image";
+          "org.opencontainers.image.description" = "Example image for ociImageLayout";
+        };
+      };
+    }
+  ];
+}
+```
diff --git a/packages/by-name/ociImageLayout/package.nix b/packages/by-name/ociImageLayout/package.nix
new file mode 100644
index 0000000000..3cbcb94f8f
--- /dev/null
+++ b/packages/by-name/ociImageLayout/package.nix
@@ -0,0 +1,40 @@
+# OCI image layout. Can be pushed to a registry or used as a local image.
+{ lib
+, runCommand
+, writers
+, nix
+}:
+{
+  # manifests is a list of ociImageManifest
+  manifests ? [ ]
+  # extraIndex is a set of additional fields to add to the index.json
+, extraIndex ? { }
+}:
+let
+  manifestDescriptors = lib.lists.map (manifest: builtins.fromJSON (builtins.readFile (manifest + "/media-descriptor.json"))) manifests;
+  index = writers.writeJSON "index.json" (
+    {
+      schemaVersion = 2;
+      mediaType = "application/vnd.oci.image.index.v1+json";
+    } // extraIndex // {
+      manifests = manifestDescriptors;
+    }
+  );
+in
+runCommand "oci-image-layout"
+{
+  buildInputs = [ nix ];
+  blobDirs = lib.lists.map (manifest: manifest + "/blobs/sha256") manifests;
+  inherit index;
+} ''
+  # add the index.json, image-layout file and all blobs to the output
+  srcs=($blobDirs)
+  mkdir -p $out/blobs/sha256
+  cp $index $out/index.json
+  echo '{"imageLayoutVersion": "1.0.0"}' > $out/image-layout
+  for src in $srcs; do
+    for blob in $(ls $src); do
+      ln -s "$(realpath $src/$blob)" "$out/blobs/sha256/$blob"
+    done
+  done
+''
diff --git a/packages/by-name/ociImageManifest/README.md b/packages/by-name/ociImageManifest/README.md
new file mode 120000
index 0000000000..4126071467
--- /dev/null
+++ b/packages/by-name/ociImageManifest/README.md
@@ -0,0 +1 @@
+../ociImageLayout/README.md
\ No newline at end of file
diff --git a/packages/by-name/ociImageManifest/package.nix b/packages/by-name/ociImageManifest/package.nix
new file mode 100644
index 0000000000..67197ae7a3
--- /dev/null
+++ b/packages/by-name/ociImageManifest/package.nix
@@ -0,0 +1,46 @@
+# application/vnd.oci.image.manifest.v1+json
+{ lib
+, ociImageConfig
+, runCommand
+, writers
+, nix
+}:
+{
+  # layers is a list of ociLayerTar
+  layers ? [ ]
+  # extraConfig is a set of extra configuration options
+, extraConfig ? { }
+  # extraManifest is a set of extra manifest options
+, extraManifest ? { }
+}:
+let
+  config = ociImageConfig { inherit layers extraConfig; };
+  configDescriptor = builtins.fromJSON (builtins.readFile (config + "/media-descriptor.json"));
+  configPlatform = builtins.fromJSON (builtins.readFile (config + "/platform.json"));
+  layerDescriptors = lib.lists.map (layer: builtins.fromJSON (builtins.readFile (layer + "/media-descriptor.json"))) layers;
+  manifest = writers.writeJSON "image-manifest.json" ({
+    schemaVersion = 2;
+    mediaType = "application/vnd.oci.image.manifest.v1+json";
+  } // extraManifest // {
+    config = configDescriptor;
+    layers = layerDescriptors;
+  });
+in
+runCommand "oci-image-manifest"
+{
+  blobDirs = lib.lists.map (layer: layer + "/blobs/sha256") (layers ++ [ config ]);
+  platformJSON = builtins.toJSON configPlatform;
+  buildInputs = [ nix ];
+  inherit manifest;
+} ''
+  mkdir -p $out/blobs/sha256
+  sha256=$(nix-hash --type sha256 --flat $manifest)
+  cp $manifest "$out/blobs/sha256/$sha256"
+  ln -s "$out/blobs/sha256/$sha256" "$out/image-manifest.json"
+  echo -n "{\"mediaType\": \"application/vnd.oci.image.manifest.v1+json\", \"size\": $(stat -c %s $manifest), \"digest\": \"sha256:$sha256\", \"platform\": $platformJSON}" > $out/media-descriptor.json
+  for src in $blobDirs; do
+    for blob in $(ls $src); do
+      ln -s "$src/$blob" "$out/blobs/sha256/$blob"
+    done
+  done
+''
diff --git a/packages/by-name/ociLayerTar/README.md b/packages/by-name/ociLayerTar/README.md
new file mode 120000
index 0000000000..4126071467
--- /dev/null
+++ b/packages/by-name/ociLayerTar/README.md
@@ -0,0 +1 @@
+../ociImageLayout/README.md
\ No newline at end of file
diff --git a/packages/by-name/ociLayerTar/package.nix b/packages/by-name/ociLayerTar/package.nix
new file mode 100644
index 0000000000..fb33623b48
--- /dev/null
+++ b/packages/by-name/ociLayerTar/package.nix
@@ -0,0 +1,63 @@
+# application/vnd.oci.image.layer.v1.tar
+# application/vnd.oci.image.layer.v1.tar+gzip
+# application/vnd.oci.image.layer.v1.tar+zstd
+{ lib, runCommandLocal, nix, gzip, zstd }:
+{
+  # files is a list of objects with the following attributes:
+  #   source: the path to the file or directory to include in the layer
+  #   destination: the path to place the file or directory in the layer
+  files ? [ ]
+  # compression is the compression algorithm to use, either "gzip" or "zstd"
+, compression ? "gzip"
+}:
+runCommandLocal "ociLayer"
+{
+  fileSources = lib.lists.map (file: file.source) files;
+  fileDestinations = lib.lists.map (file: file.destination or file.source) files;
+  outPath = "layer" + (
+    if compression == "gzip" then ".tar.gz"
+    else if compression == "zstd" then ".tar.zst"
+    else ".tar"
+  );
+  mediaType = "application/vnd.oci.image.layer.v1.tar" + (if compression == "" then "" else "+" + compression);
+  nativeBuildInputs = [ nix ]
+    ++ lib.optional (compression == "gzip") gzip
+    ++ lib.optional (compression == "zstd") zstd;
+  inherit compression;
+} ''
+  set -o pipefail
+  srcs=($fileSources)
+  dests=($fileDestinations)
+  mkdir -p ./root $out
+
+  # Copy files into the tree (./root/)
+  for i in ''${!srcs[@]}; do
+      mkdir -p "./root/$(dirname ''${dests[$i]})"
+      cp -rT "''${srcs[i]}" "./root/''${dests[$i]}"
+  done
+
+  # Create the layer tarball
+  tar --sort=name --owner=root:0 --group=root:0 --mode=544 --mtime='UTC 1970-01-01' -cC ./root -f $out/layer.tar .
+  # Calculate the layer tarball's diffID (hash of the uncompressed tarball)
+  diffID=$(nix-hash --type sha256 --flat $out/layer.tar)
+  # Compress the layer tarball
+  if [[ "$compression" = "gzip" ]]; then
+    gzip -c $out/layer.tar > $out/$outPath
+  elif [[ "$compression" = "zstd" ]]; then
+    zstd -T0 -q -c $out/layer.tar > $out/$outPath
+  else
+    mv $out/layer.tar $out/$outPath
+  fi
+  rm -f $out/layer.tar
+
+  # Calculate the blob's sha256 hash and write the media descriptor
+  sha256=$(nix-hash --type sha256 --flat $out/$outPath)
+  echo -n "{\"mediaType\": \"$mediaType\", \"size\": $(stat -c %s $out/$outPath), \"digest\": \"sha256:$sha256\"}" > $out/media-descriptor.json
+  echo -n "sha256:$diffID" > $out/DiffID
+
+  # Move the compressed layer tarball to the blobs directory and create a symlink
+  mkdir -p $out/blobs/sha256
+  mv $out/$outPath $out/blobs/sha256/$sha256
+  ln -s $out/blobs/sha256/$sha256 $out/$outPath
+  rm -rf ./root
+''