-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
nix: create efficient oci images with reusable layers
This set of functions generates layers that can be combined into OCI image layout directories. Those can directly be pushed to a registry. Strong focus was placed on making the layers reusable in arbitrary ways. This allows for efficient distribution of binary artifacts via container image layers.
- Loading branch information
Showing
9 changed files
with
247 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
../ociImageLayout/README.md |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
# application/vnd.oci.image.config.v1+json | ||
{ lib, runCommand, writers, nix }: | ||
# layers is a list of ociLayerTar | ||
# extraConfig is a set of extra configuration options | ||
{ layers ? [ ] | ||
, extraConfig ? { } | ||
}: | ||
let | ||
diffIDs = lib.lists.map (layer: builtins.readFile (layer + "/DiffID")) layers; | ||
config = { | ||
architecture = "amd64"; | ||
os = "linux"; | ||
} // extraConfig // { | ||
rootfs = { type = "layers"; diff_ids = diffIDs; }; | ||
}; | ||
configJSON = writers.writeJSON "image-config.json" config; | ||
in | ||
runCommand "oci-image-config" | ||
{ | ||
buildInputs = [ nix ]; | ||
platformJSON = builtins.toJSON { inherit (config) architecture; inherit (config) os; }; | ||
inherit configJSON; | ||
} '' | ||
# write the config to a file under blobs/sha256 | ||
mkdir -p $out/blobs/sha256 | ||
sha256=$(nix-hash --type sha256 --flat $configJSON) | ||
cp $configJSON "$out/blobs/sha256/$sha256" | ||
# create a symlink to the image config | ||
ln -s "$out/blobs/sha256/$sha256" "$out/image-config.json" | ||
# write the platform.json | ||
echo "$platformJSON" > "$out/platform.json" | ||
# write the media descriptor | ||
echo -n "{\"mediaType\": \"application/vnd.oci.image.config.v1+json\", \"size\": $(stat -c %s $configJSON), \"digest\": \"sha256:$sha256\"}" > $out/media-descriptor.json | ||
'' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,55 @@ | ||
# OCI image tools | ||
|
||
This is a set of nix functions for creating reproducible and cachable multi-layer OCI images. | ||
|
||
It uses the following functions: | ||
|
||
- `ociImageLayout`: Top level function for creating an [OCI image directory layout](https://github.com/opencontainers/image-spec/blob/v1.1.0/image-layout.md). This can be used directly (by podman) or uploaded to a registry (`crane push path/to/directory registry/image/name:tag`). | ||
- `ociImageManifest`: An [OCI image manifest](https://github.com/opencontainers/image-spec/blob/v1.1.0/manifest.md#image-manifest) that can be added to the top-level layout. A manifest contains a configuration and layers. | ||
- `ociImageConfig`: An [OCI image configuration](https://github.com/opencontainers/image-spec/blob/v1.1.0/config.md) that can be included in a manifest. The configuration describdes the image including layers, entrypoint, arguments, architecture, os and more. | ||
- `ociLayerTar`: An [OCI image layer filesystem changeset (layer tar)](https://github.com/opencontainers/image-spec/blob/v1.1.0/layer.md). Contains an individual container image layer. Can be freely remixed with other layers. Takes a list of store paths and their target destinations in the image. | ||
|
||
## Example | ||
|
||
The following example creates an image containing two layers: | ||
|
||
- one layer for nginx | ||
- one layer with bash, jq, a script and configuration | ||
|
||
```nix | ||
ociImageLayout { | ||
manifests = [ | ||
ociImageManifest | ||
{ | ||
layers = [ | ||
ociLayerTar { | ||
files = [ { source = nginx; } ]; | ||
} | ||
ociLayerTar { | ||
files = [ | ||
{ source = bash; destination = "/bin/bash"; } | ||
{ source = jq; destination = "/bin/jq"; } | ||
{ source = writeShellScript "entrypoint.sh" '' jq $CONFIG_PATH ; nginx -g 'daemon off;' ''; destination = "/entrypoint.sh"; } | ||
{ source = writers.writeJSON "conf.json" { a = 1; b = 2; }; destination = "/etc/configuration.json"; } | ||
]; | ||
} | ||
]; | ||
extraConfig = { | ||
"config" = { | ||
"Env" = [ | ||
"PATH=/bin:/usr/bin" | ||
"CONFIG_PATH=/config" | ||
]; | ||
"Entrypoint" = [ "/entrypoint.sh" ]; | ||
}; | ||
}; | ||
extraManifest = { | ||
"annotations" = { | ||
"org.opencontainers.image.title" = "example-image"; | ||
"org.opencontainers.image.description" = "Example image for ociImageLayout"; | ||
}; | ||
}; | ||
} | ||
]; | ||
} | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
# OCI image layout. Can be pushed to a registry or used as a local image. | ||
{ lib | ||
, runCommand | ||
, writers | ||
, nix | ||
}: | ||
# manifests is a list of ociImageManifest | ||
# | ||
# extraIndex is a set of additional fields to add to the index.json | ||
{ manifests ? [ ] | ||
, extraIndex ? { } | ||
}: | ||
let | ||
manifestDescriptors = lib.lists.map (manifest: builtins.fromJSON (builtins.readFile (manifest + "/media-descriptor.json"))) manifests; | ||
index = writers.writeJSON "index.json" ( | ||
{ | ||
schemaVersion = 2; | ||
mediaType = "application/vnd.oci.image.index.v1+json"; | ||
} // extraIndex // { | ||
manifests = manifestDescriptors; | ||
} | ||
); | ||
in | ||
runCommand "oci-image-layout" | ||
{ | ||
buildInputs = [ nix ]; | ||
blobDirs = lib.lists.map (manifest: manifest + "/blobs/sha256") manifests; | ||
inherit index; | ||
} '' | ||
# add the index.json, image-layout file and all blobs to the output | ||
srcs=($blobDirs) | ||
mkdir -p $out/blobs/sha256 | ||
cp $index $out/index.json | ||
echo '{"imageLayoutVersion": "1.0.0"}' > $out/image-layout | ||
for src in $srcs; do | ||
for blob in $(ls $src); do | ||
ln -s "$(realpath $src/$blob)" "$out/blobs/sha256/$blob" | ||
done | ||
done | ||
'' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
../ociImageLayout/README.md |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
# application/vnd.oci.image.manifest.v1+json | ||
{ lib | ||
, ociImageConfig | ||
, runCommand | ||
, writers | ||
, nix | ||
}: | ||
# layers is a list of ociLayerTar | ||
# extraConfig is a set of extra configuration options | ||
# extraManifest is a set of extra manifest options | ||
{ layers ? [ ] | ||
, extraConfig ? { } | ||
, extraManifest ? { } | ||
}: | ||
let | ||
config = ociImageConfig { inherit layers extraConfig; }; | ||
configDescriptor = builtins.fromJSON (builtins.readFile (config + "/media-descriptor.json")); | ||
configPlatform = builtins.fromJSON (builtins.readFile (config + "/platform.json")); | ||
layerDescriptors = lib.lists.map (layer: builtins.fromJSON (builtins.readFile (layer + "/media-descriptor.json"))) layers; | ||
manifest = writers.writeJSON "image-manifest.json" ( | ||
{ | ||
schemaVersion = 2; | ||
mediaType = "application/vnd.oci.image.manifest.v1+json"; | ||
} // extraManifest // { | ||
config = configDescriptor; | ||
layers = layerDescriptors; | ||
} | ||
); | ||
in | ||
runCommand "oci-image-manifest" | ||
{ | ||
blobDirs = lib.lists.map (layer: layer + "/blobs/sha256") (layers ++ [ config ]); | ||
platformJSON = builtins.toJSON configPlatform; | ||
buildInputs = [ nix ]; | ||
inherit manifest; | ||
} '' | ||
mkdir -p $out/blobs/sha256 | ||
sha256=$(nix-hash --type sha256 --flat $manifest) | ||
cp $manifest "$out/blobs/sha256/$sha256" | ||
ln -s "$out/blobs/sha256/$sha256" "$out/image-manifest.json" | ||
echo -n "{\"mediaType\": \"application/vnd.oci.image.manifest.v1+json\", \"size\": $(stat -c %s $manifest), \"digest\": \"sha256:$sha256\", \"platform\": $platformJSON}" > $out/media-descriptor.json | ||
for src in $blobDirs; do | ||
for blob in $(ls $src); do | ||
ln -s "$src/$blob" "$out/blobs/sha256/$blob" | ||
done | ||
done | ||
'' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
../ociImageLayout/README.md |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
# application/vnd.oci.image.layer.v1.tar | ||
# application/vnd.oci.image.layer.v1.tar+gzip | ||
# application/vnd.oci.image.layer.v1.tar+zstd | ||
{ lib, runCommandLocal, nix, gzip, zstd }: | ||
# files is a list of objects with the following attributes: | ||
# source: the path to the file or directory to include in the layer | ||
# destination: the path to place the file or directory in the layer | ||
# | ||
# compression is the compression algorithm to use, either "gzip" or "zstd" | ||
{ files ? [ ] | ||
, compression ? "gzip" | ||
}: | ||
runCommandLocal "ociLayer" | ||
{ | ||
fileSources = lib.lists.map (file: file.source) files; | ||
fileDestinations = lib.lists.map (file: file.destination or file.source) files; | ||
outPath = "layer" + ( | ||
if compression == "gzip" then ".tar.gz" | ||
else if compression == "zstd" then ".tar.zst" | ||
else ".tar" | ||
); | ||
mediaType = "application/vnd.oci.image.layer.v1.tar" + (if compression == "" then "" else "+" + compression); | ||
nativeBuildInputs = [ nix ] | ||
++ lib.optional (compression == "gzip") gzip | ||
++ lib.optional (compression == "zstd") zstd; | ||
inherit compression; | ||
} '' | ||
set -o pipefail | ||
srcs=($fileSources) | ||
dests=($fileDestinations) | ||
mkdir -p ./root $out | ||
# Copy files into the tree (./root/) | ||
for i in ''${!srcs[@]}; do | ||
mkdir -p "./root/$(dirname ''${dests[$i]})" | ||
cp -rT "''${srcs[i]}" "./root/''${dests[$i]}" | ||
done | ||
# Create the layer tarball | ||
tar --sort=name --owner=root:0 --group=root:0 --mode=544 --mtime='UTC 1970-01-01' -cC ./root -f $out/layer.tar . | ||
# Calculate the layer tarball's diffID (hash of the uncompressed tarball) | ||
diffID=$(nix-hash --type sha256 --flat $out/layer.tar) | ||
# Compress the layer tarball | ||
if [[ "$compression" = "gzip" ]]; then | ||
gzip -c $out/layer.tar > $out/$outPath | ||
elif [[ "$compression" = "zstd" ]]; then | ||
zstd -T0 -q -c $out/layer.tar > $out/$outPath | ||
else | ||
mv $out/layer.tar $out/$outPath | ||
fi | ||
rm -f $out/layer.tar | ||
# Calculate the blob's sha256 hash and write the media descriptor | ||
sha256=$(nix-hash --type sha256 --flat $out/$outPath) | ||
echo -n "{\"mediaType\": \"$mediaType\", \"size\": $(stat -c %s $out/$outPath), \"digest\": \"sha256:$sha256\"}" > $out/media-descriptor.json | ||
echo -n "sha256:$diffID" > $out/DiffID | ||
# Move the compressed layer tarball to the blobs directory and create a symlink | ||
mkdir -p $out/blobs/sha256 | ||
mv $out/$outPath $out/blobs/sha256/$sha256 | ||
ln -s $out/blobs/sha256/$sha256 $out/$outPath | ||
rm -rf ./root | ||
'' |