diff --git a/docs/styles/config/vocabularies/edgeless/accept.txt b/docs/styles/config/vocabularies/edgeless/accept.txt index 9610913be8..061016f285 100644 --- a/docs/styles/config/vocabularies/edgeless/accept.txt +++ b/docs/styles/config/vocabularies/edgeless/accept.txt @@ -13,7 +13,9 @@ backport Bazel bootloader Bootstrapper +cachable cachix +changeset cloud cmdline config @@ -50,6 +52,7 @@ iodepth IPSec Istio journald +jq Kata KEK KMS @@ -66,6 +69,7 @@ Nginx paravisor PCR plaintext +podman protobuf proxied QEMU diff --git a/packages/by-name/ociImageConfig/README.md b/packages/by-name/ociImageConfig/README.md new file mode 120000 index 0000000000..4126071467 --- /dev/null +++ b/packages/by-name/ociImageConfig/README.md @@ -0,0 +1 @@ +../ociImageLayout/README.md \ No newline at end of file diff --git a/packages/by-name/ociImageConfig/package.nix b/packages/by-name/ociImageConfig/package.nix new file mode 100644 index 0000000000..80367bd46f --- /dev/null +++ b/packages/by-name/ociImageConfig/package.nix @@ -0,0 +1,30 @@ +# application/vnd.oci.image.config.v1+json +{ lib, runCommand, writers, nix }: +# layers is a list of ociLayerTar +# extraConfig is a set of extra configuration options +{ layers ? [ ] +, extraConfig ? { } +}: +let + diffIDs = lib.lists.map (layer: builtins.readFile (layer + "/DiffID")) layers; + config = { + architecture = "amd64"; + os = "linux"; + } // extraConfig // { + rootfs = { type = "layers"; diff_ids = diffIDs; }; + }; + configJSON = writers.writeJSON "image-config.json" config; +in +runCommand "oci-image-config" +{ + buildInputs = [ nix ]; + platformJSON = builtins.toJSON { inherit (config) architecture; inherit (config) os; }; + inherit configJSON; +} '' + mkdir -p $out/blobs/sha256 + sha256=$(nix-hash --type sha256 --flat $configJSON) + cp $configJSON "$out/blobs/sha256/$sha256" + ln -s "$out/blobs/sha256/$sha256" "$out/image-config.json" + echo "$platformJSON" > "$out/platform.json" + echo -n "{\"mediaType\": \"application/vnd.oci.image.config.v1+json\", \"size\": $(stat -c %s $configJSON), \"digest\": \"sha256:$sha256\"}" > $out/media-descriptor.json +'' diff --git a/packages/by-name/ociImageLayout/README.md b/packages/by-name/ociImageLayout/README.md new file mode 100644 index 0000000000..dcf378dcd3 --- /dev/null +++ b/packages/by-name/ociImageLayout/README.md @@ -0,0 +1,55 @@ +# OCI image tools + +This is a set of nix functions for creating reproducible and cachable multi-layer OCI images. + +It uses the following functions: + +- `ociImageLayout`: Top level function for creating an [OCI image directory layout](https://github.com/opencontainers/image-spec/blob/v1.1.0/image-layout.md). This can be used directly (by podman) or uploaded to a registry (`crane push path/to/directory registry/image/name:tag`). +- `ociImageManifest`: An OCI image manifest(https://github.com/opencontainers/image-spec/blob/v1.1.0/manifest.md#image-manifest) that can be added to the top-level layout. A manifest contains a configuration and layers. +- `ociImageConfig`: An OCI image configuration(https://github.com/opencontainers/image-spec/blob/v1.1.0/config.md) that can be included in a manifest. The configuration describdes the image including layers, entrypoint, arguments, architecture, os and more. +- `ociLayerTar`: An OCI image layer filesystem changeset (layer tar)(https://github.com/opencontainers/image-spec/blob/v1.1.0/layer.md). Contains an individual container image layer. Can be freely remixed with other layers. Takes a list of store paths and their target destinations in the image. + +## Example + +The following example creates an image containing two layers: + +- one layer for nginx +- one layer with bash, jq, a script and configuration + +```nix +ociImageLayout { + manifests = [ + ociImageManifest + { + layers = [ + ociLayerTar { + files = [ { source = nginx; } ]; + } + ociLayerTar { + files = [ + { source = bash; destination = "/bin/bash"; } + { source = jq; destination = "/bin/jq"; } + { source = writeShellScript "entrypoint.sh" '' jq $CONFIG_PATH ; nginx -g 'daemon off;' ''; destination = "/entrypoint.sh"; } + { source = writers.writeJSON "conf.json" { a = 1; b = 2; }; destination = "/etc/configuration.json"; } + ]; + } + ]; + extraConfig = { + "config" = { + "Env" = [ + "PATH=/bin:/usr/bin" + "CONFIG_PATH=/config" + ]; + "Entrypoint" = [ "/entrypoint.sh" ]; + }; + }; + extraManifest = { + "annotations" = { + "org.opencontainers.image.title" = "example-image"; + "org.opencontainers.image.description" = "Example image for ociImageLayout"; + }; + }; + } + ]; +} +``` diff --git a/packages/by-name/ociImageLayout/package.nix b/packages/by-name/ociImageLayout/package.nix new file mode 100644 index 0000000000..b2ae3c9469 --- /dev/null +++ b/packages/by-name/ociImageLayout/package.nix @@ -0,0 +1,39 @@ +# OCI image layout. Can be pushed to a registry or used as a local image. +{ lib +, runCommand +, writers +, nix +}: +# manifests is a list of ociImageManifest +# +# extraIndex is a set of additional fields to add to the index.json +{ manifests ? [ ] +, extraIndex ? { } +}: +let + manifestDescriptors = lib.lists.map (manifest: builtins.fromJSON (builtins.readFile (manifest + "/media-descriptor.json"))) manifests; + index = writers.writeJSON "index.json" ( + { + schemaVersion = 2; + mediaType = "application/vnd.oci.image.index.v1+json"; + } // extraIndex // { + manifests = manifestDescriptors; + } + ); +in +runCommand "oci-image-layout" +{ + buildInputs = [ nix ]; + blobDirs = lib.lists.map (manifest: manifest + "/blobs/sha256") manifests; + inherit index; +} '' + srcs=($blobDirs) + mkdir -p $out/blobs/sha256 + cp $index $out/index.json + echo '{"imageLayoutVersion": "1.0.0"}' > $out/image-layout + for src in $srcs; do + for blob in $(ls $src); do + ln -s "$(realpath $src/$blob)" "$out/blobs/sha256/$blob" + done + done +'' diff --git a/packages/by-name/ociImageManifest/README.md b/packages/by-name/ociImageManifest/README.md new file mode 120000 index 0000000000..4126071467 --- /dev/null +++ b/packages/by-name/ociImageManifest/README.md @@ -0,0 +1 @@ +../ociImageLayout/README.md \ No newline at end of file diff --git a/packages/by-name/ociImageManifest/package.nix b/packages/by-name/ociImageManifest/package.nix new file mode 100644 index 0000000000..a5090692e0 --- /dev/null +++ b/packages/by-name/ociImageManifest/package.nix @@ -0,0 +1,47 @@ +# application/vnd.oci.image.manifest.v1+json +{ lib +, ociImageConfig +, runCommand +, writers +, nix +}: +# layers is a list of ociLayerTar +# extraConfig is a set of extra configuration options +# extraManifest is a set of extra manifest options +{ layers ? [ ] +, extraConfig ? { } +, extraManifest ? { } +}: +let + config = ociImageConfig { inherit layers extraConfig; }; + configDescriptor = builtins.fromJSON (builtins.readFile (config + "/media-descriptor.json")); + configPlatform = builtins.fromJSON (builtins.readFile (config + "/platform.json")); + layerDescriptors = lib.lists.map (layer: builtins.fromJSON (builtins.readFile (layer + "/media-descriptor.json"))) layers; + manifest = writers.writeJSON "image-manifest.json" ( + { + schemaVersion = 2; + mediaType = "application/vnd.oci.image.manifest.v1+json"; + } // extraManifest // { + config = configDescriptor; + layers = layerDescriptors; + } + ); +in +runCommand "oci-image-manifest" +{ + blobDirs = lib.lists.map (layer: layer + "/blobs/sha256") (layers ++ [ config ]); + platformJSON = builtins.toJSON configPlatform; + buildInputs = [ nix ]; + inherit manifest; +} '' + mkdir -p $out/blobs/sha256 + sha256=$(nix-hash --type sha256 --flat $manifest) + cp $manifest "$out/blobs/sha256/$sha256" + ln -s "$out/blobs/sha256/$sha256" "$out/image-manifest.json" + echo -n "{\"mediaType\": \"application/vnd.oci.image.manifest.v1+json\", \"size\": $(stat -c %s $manifest), \"digest\": \"sha256:$sha256\", \"platform\": $platformJSON}" > $out/media-descriptor.json + for src in $blobDirs; do + for blob in $(ls $src); do + ln -s "$src/$blob" "$out/blobs/sha256/$blob" + done + done +'' diff --git a/packages/by-name/ociLayerTar/README.md b/packages/by-name/ociLayerTar/README.md new file mode 120000 index 0000000000..4126071467 --- /dev/null +++ b/packages/by-name/ociLayerTar/README.md @@ -0,0 +1 @@ +../ociImageLayout/README.md \ No newline at end of file diff --git a/packages/by-name/ociLayerTar/package.nix b/packages/by-name/ociLayerTar/package.nix new file mode 100644 index 0000000000..8ffde3e472 --- /dev/null +++ b/packages/by-name/ociLayerTar/package.nix @@ -0,0 +1,63 @@ +# application/vnd.oci.image.layer.v1.tar +# application/vnd.oci.image.layer.v1.tar+gzip +# application/vnd.oci.image.layer.v1.tar+zstd +{ lib, runCommandLocal, nix, gzip, zstd }: +# files is a list of objects with the following attributes: +# source: the path to the file or directory to include in the layer +# destination: the path to place the file or directory in the layer +# +# compression is the compression algorithm to use, either "gzip" or "zstd" +{ files ? [ ] +, compression ? "gzip" +}: +runCommandLocal "ociLayer" +{ + fileSources = lib.lists.map (file: file.source) files; + fileDestinations = lib.lists.map (file: file.destination or file.source) files; + outPath = "layer" + ( + if compression == "gzip" then ".tar.gz" + else if compression == "zstd" then ".tar.zst" + else ".tar" + ); + mediaType = "application/vnd.oci.image.layer.v1.tar" + (if compression == "" then "" else "+" + compression); + nativeBuildInputs = [ nix ] + ++ lib.optional (compression == "gzip") gzip + ++ lib.optional (compression == "zstd") zstd; + inherit compression; +} '' + set -o pipefail + srcs=($fileSources) + dests=($fileDestinations) + mkdir -p ./root $out + + # Copy files into the tree (./root/) + for i in ''${!srcs[@]}; do + mkdir -p "./root/$(dirname ''${dests[$i]})" + cp -rT "''${srcs[i]}" "./root/''${dests[$i]}" + done + + # Create the layer tarball + tar --sort=name --owner=root:0 --group=root:0 --mode=544 --mtime='UTC 1970-01-01' -cC ./root -f $out/layer.tar . + # Calculate the layer tarball's diffID (hash of the uncompressed tarball) + diffID=$(nix-hash --type sha256 --flat $out/layer.tar) + # Compress the layer tarball + if [[ "$compression" = "gzip" ]]; then + gzip -c $out/layer.tar > $out/$outPath + elif [[ "$compression" = "zstd" ]]; then + zstd -T0 -q -c $out/layer.tar > $out/$outPath + else + mv $out/layer.tar $out/$outPath + fi + rm -f $out/layer.tar + + # Calculate the blob's sha256 hash and write the media descriptor + sha256=$(nix-hash --type sha256 --flat $out/$outPath) + echo -n "{\"mediaType\": \"$mediaType\", \"size\": $(stat -c %s $out/$outPath), \"digest\": \"sha256:$sha256\"}" > $out/media-descriptor.json + echo -n "sha256:$diffID" > $out/DiffID + + # Move the compressed layer tarball to the blobs directory and create a symlink + mkdir -p $out/blobs/sha256 + mv $out/$outPath $out/blobs/sha256/$sha256 + ln -s $out/blobs/sha256/$sha256 $out/$outPath + rm -rf ./root +''