From bcb1871b1c1f9a1e78ff86b7f8120dd2c08ac266 Mon Sep 17 00:00:00 2001
From: "David E. Wheeler" <david@justatheory.com>
Date: Tue, 18 Jun 2024 18:56:04 -0400
Subject: [PATCH] POC pushing & pulling trunk from an OSI registry

Modify `trunk.mk` to build JSON files required to build [OCI image
manifests] and an [OCI image index], including new `make` variables
`TITLE`, `DESCRIPTION`, `VENDOR`, `URL`, and `REPO_URL`.

Add `push_trunk`, a shell script to take those JSON files and use `oras` to
build the images and manifests and push them to a registry.

Add `docker_compose.yml` to spin up a PGXN tools image for AMD64 and
running Postgres 16, as well as [zot], a vendor-neutral OCI registry.

Finally, modify `install_trunk` to download an artifact from a registry
address, rather than build from a local file. Use `oras` to fetch the
image for the current platform. There's no support for distinguishing
between artifacts built for different Postgres versions, but there are
annotations in the image index that would allow it.

  [OCI image manifests]: https://github.com/opencontainers/image-spec/blob/main/manifest.md
  [OCI image index]: https://github.com/opencontainers/image-spec/blob/main/image-index.md
  [zot]: https://github.com/project-zot/zot
---
 .gitignore         |   1 +
 Makefile           |   2 +
 docker-compose.yml |  57 +++++++++++++++++++++++
 install_trunk      |  17 +++++--
 push_trunk         | 114 +++++++++++++++++++++++++++++++++++++++++++++
 trunk.mk           |  58 ++++++++++++++++++++++-
 6 files changed, 243 insertions(+), 6 deletions(-)
 create mode 100644 docker-compose.yml
 create mode 100755 push_trunk

diff --git a/.gitignore b/.gitignore
index 011f240..44cc862 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,6 +9,7 @@ regression.diffs
 regression.out
 /sql/semver--?.??.?.sql
 /semver-*
+/semver_annotations.json
 /latest-changes.md
 /semver_binary_copy.bin
 /.vscode
diff --git a/Makefile b/Makefile
index 2dd11c6..28187f7 100644
--- a/Makefile
+++ b/Makefile
@@ -6,6 +6,8 @@ DISTVERSION  = $(shell grep -m 1 '[[:space:]]\{3\}"version":' META.json | \
                sed -e 's/[[:space:]]*"version":[[:space:]]*"\([^"]*\)",\{0,1\}/\1/')
 
 MODULEDIR    = $(EXTENSION)
+DESCRIPTION  = A Postgres data type for the Semantic Version format with support for btree and hash indexing.
+REPO_URL     = https://github.com/theory/pg-semver
 DATA         = $(wildcard sql/*--*.sql)
 DOCS         = $(wildcard doc/*.mmd)
 TESTS        = $(wildcard test/sql/*.sql)
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000..b1b6d61
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,57 @@
+version: "3.9"
+name: trunking
+
+# Start a Zot service for pushing and pulling OCI images, and a
+# pgxn/pgxn-tools image for building and testing Linux/AMD binaries.
+#
+# docker compose up -d
+# docker compose exec linux bash
+# make clean && make && make trunk
+#
+# On macOS:
+#   make clean && make && make trunk
+#   ./push_trunk localhost:5000/theory/semver semver-0.32.1+pg16-darwin-23.5.0-arm64 semver-0.32.1+pg16-linux-amd64
+#   clean -dfx --exclude=.vscode
+#   find "$(pg_config --sharedir)" "$(pg_config --pkglibdir)" "$(pg_config --docdir)" -name '*semver*'  -exec rm -rf "{}" \;
+#
+#   ./install_trunk localhost:5000/theory/semver:v1
+#   find "$(pg_config --sharedir)" "$(pg_config --pkglibdir)" "$(pg_config --docdir)" -name '*semver*'
+#
+# Back on Linux
+#   ./install_trunk zot:5000/theory/semver:v1
+#   find "$(pg_config --sharedir)" "$(pg_config --pkglibdir)" "$(pg_config --docdir)" -name '*semver*'
+
+# Name the network for all of the services to join.
+networks:
+  default:
+    name: pgxnnet
+
+services:
+  zot:
+    image: ghcr.io/project-zot/zot-linux-arm64:latest
+    container_name: zot
+    ports:
+      - 5000:5000
+    hostname: zot
+
+  linux:
+    image: pgxn/pgxn-tools
+    platform: "linux/amd64"
+    container_name: linux
+    hostname: linux
+    working_dir: /work
+    volumes:
+      - ".:/work"
+    # Install oras, then start Postgres 16 and install rsync and jq
+    entrypoint:
+      - "/bin/sh"
+      - -ecx
+      - |
+          cd /tmp
+          curl -LO "https://github.com/oras-project/oras/releases/download/v1.2.0/oras_1.2.0_linux_amd64.tar.gz"
+          mkdir -p oras-install/
+          tar -zxf oras_1.2.0_*.tar.gz -C oras-install/
+          sudo mv oras-install/oras /usr/local/bin/
+          rm -rf oras_1.2.0_*.tar.gz oras-install/
+          pg-start 16 rsync jq
+          tail -f /dev/null
diff --git a/install_trunk b/install_trunk
index 7b42b00..78521f4 100755
--- a/install_trunk
+++ b/install_trunk
@@ -22,18 +22,27 @@ install_trunk() {
     fi
 
     # Determine the platform.
-    local my_os my_osv my_arch
+    local my_os my_osv my_arch platform
     my_os=$(uname | tr '[:upper:]' '[:lower:]')
     my_osv=$(uname -r)
     my_arch="$(uname -m)"
     if [ "$my_arch" == "x86_64" ]; then my_arch=amd64; fi
+    if [ "$my_os" == "darwin" ]; then
+        platform="$my_os/$my_arch:$my_osv"
+    else
+        platform="$my_os/$my_arch"
+    fi
+
+    # Download.
+    local file
+    file=$(basename "$(oras pull --no-tty --plain-http  --format 'go-template={{(first .files).path}}' --platform "$platform" "$trunk")")
 
     # Unpack.
-    printf "Unpacking %s\n" "$trunk"
-    tar zxf "$trunk"
+    printf "Unpacking %s\n" "$file"
+    tar zxf "$file"
 
     # Go into the directory
-    local dir="${trunk%.*}"
+    local dir="${file%.*}"
     cd "$dir" || exit
 
     # Verify the checksums.
diff --git a/push_trunk b/push_trunk
new file mode 100755
index 0000000..b36d574
--- /dev/null
+++ b/push_trunk
@@ -0,0 +1,114 @@
+#!/bin/bash
+
+# POC for publishing Trunk packages to an OCI repository with an image index
+# to allow pulling a platform-specific binary. Requires:
+#
+# * oras
+# * jq
+# * zot: docker run -d -p 5000:5000 --name oras-quickstart ghcr.io/project-zot/zot-linux-arm64:latest
+#
+# Inspired by the Homebrew implementation of the pattern as referenced in
+# https://github.com/oras-project/oras/issues/237, and with thanks to the
+# denizens of the #oras and #zot channels on the CNCF Slack.
+
+trap 'exit' ERR
+set -E
+
+# OCI types to create.
+ARTIFACT_TYPE=application/vnd.pgxn.trunk.layer.v1
+MEDIA_TYPE=application/vnd.oci.image.layer.v1.tar+gzip
+CONFIG_TYPE=application/vnd.oci.image.config.v1+json
+OCI_DIR=oci_dir
+INDEX_FILE=image_index.json
+
+push_image() {
+    # Push the image into the OCI layout directory $OCI_DIR.
+    local trunk=$1
+    oras push --no-tty \
+        --oci-layout "$OCI_DIR" \
+        --artifact-type "${ARTIFACT_TYPE}" \
+        --config "${trunk}_config.json":"$CONFIG_TYPE" \
+        --format go-template='{{.digest}}' \
+        --annotation-file "${trunk}_annotations.json" \
+        "$trunk.trunk":"$MEDIA_TYPE"
+}
+
+make_manifest() {
+    local trunk=$1
+    local digest=$2
+    local anno platform
+
+    # Extract just the pgxn.trunk annotations.
+    anno=$(jq -c \
+        '.["$manifest"] | with_entries(select(.key | startswith("org.pgxn.trunk.")))' \
+        "${trunk}_annotations.json"
+    )
+
+    # Extract the platform config.
+    platform=$(jq \
+        'pick(.os, .["os.version"], .architecture)| with_entries(select(.value |. !=null and . != ""))' \
+        "$trunk"_config.json
+    )
+
+    # Create and return the image manifest.
+    oras manifest fetch --oci-layout "$OCI_DIR@${digest}" --descriptor \
+        | jq --argjson anno "$anno" --argjson platform "$platform" \
+        '{
+            mediaType: .mediaType,
+            size: .size,
+            digest: .digest,
+            platform: $platform,
+            annotations: $anno
+        }'
+}
+
+write_index() {
+    darwin_manifest=$1
+    linux_manifest=$2
+
+    # Build the image index with the two manifests.
+    jq -n --argjson linux "$linux_manifest" \
+        --argjson darwin "$darwin_manifest" \
+        --argjson annotations "$(cat semver_annotations.json)" \
+        '{
+            schemaVersion: 2,
+            mediaType: "application/vnd.oci.image.index.v1+json",
+            manifests: [$linux, $darwin],
+            annotations: $annotations
+        }' > "$INDEX_FILE"
+}
+
+push_trunk() {
+    # Only testing for Darwin and Linux rn.
+    local repo=$1
+    local darwin_trunk=$2
+    local linux_trunk=$3
+
+    if [ -z "$repo" ] || [ -z "$darwin_trunk" ] || [ -z "$linux_trunk" ]; then
+        printf "Usage:\n\n  %s REPO DARWIN.trunk LINUX.trunk\n" "$0"
+        exit 1
+    fi
+
+    # Push the images and grab the resulting digests.
+    darwin_digest=$(push_image "$darwin_trunk")
+    linux_digest=$(push_image "$linux_trunk")
+
+    # Create the image manifests.
+    darwin_manifest=$(make_manifest "$darwin_trunk" "$darwin_digest")
+    linux_manifest=$(make_manifest "$linux_trunk" "$linux_digest")
+
+    # Write out and push the image index.
+    write_index "$darwin_manifest" "$linux_manifest"
+    oras manifest push --oci-layout ./"$OCI_DIR":image-index "$INDEX_FILE"
+
+    # Push everything from the local layout to the remote registry.
+    oras cp --from-oci-layout ./"$OCI_DIR":image-index --to-plain-http "${repo}:v1"
+
+    # View the remote image index manifest.
+    oras manifest get --plain-http "${repo}:v1" | jq
+
+    # Cleanup.
+    rm -rf "$OCI_DIR" "$INDEX_FILE"
+}
+
+push_trunk "$@"
diff --git a/trunk.mk b/trunk.mk
index 4a23894..b2ec09c 100644
--- a/trunk.mk
+++ b/trunk.mk
@@ -1,10 +1,16 @@
 # Trunk packaging. To use, create a standard PGXS Makefile for your extension.
-# The only extra variables are DISTVERSION, LICENSE, and LANGUAGE.
+# The only extra variables are DISTVERSION, TITLE, DESCRIPTION, LICENSE,
+# LANGUAGE, VENDOR, URL, and REPO_URL.
 #
 # ``` make
 # DISTVERSION  = 1.2.1
+# TITLE        = Bike
+# DESCRIPTION  = A bicycle inside your database.
 # LICENSE      = mit
 # LANGUAGE     = c
+# VENDOR       = PGXN
+# URL          = https://pgxn.org/dist/bike
+# REPO_URL     = https://github.com/pgxn/bike
 #
 # EXTENSION    = bike
 # MODULEDIR    = $(EXTENSION)
@@ -38,6 +44,9 @@
 
 LAUNGUAGE      ?= c
 LICENSE        ?= PostgreSQL
+TITLE          ?= $(EXTENSION)
+VENDOR         ?= PGXN
+URL            ?= $(REPO_URL)
 
 pkg_arch       := $(shell uname -m)
 ifeq ($(pkg_arch),x86_64)
@@ -64,7 +73,52 @@ pkg_info_files ?= $(wildcard README* readme* Readme* LICENSE* license* License*
 EXTRA_CLEAN    += $(EXTENSION)-$(DISTVERSION)+*/
 
 # Phony target to create the trunk and OCI JSON files.
-trunk: $(pkg).trunk
+trunk: $(pkg).trunk $(pkg)_config.json $(pkg)_annotations.json $(EXTENSION)_annotations.json
+
+# Use jq to create the OCI image index annoations.
+# https://github.com/opencontainers/image-spec/blob/main/annotations.md
+$(EXTENSION)_annotations.json:
+	jq -n \
+	    --arg org.opencontainers.image.created "$$(date +%Y-%m-%dT%TZ)" \
+		--arg org.opencontainers.image.licenses "$(LICENSE)" \
+		--arg org.opencontainers.image.title "$(TITLE)" \
+		--arg org.opencontainers.image.description "$(DESCRIPTION)" \
+		--arg org.opencontainers.image.source "$(REPO_URL)" \
+		--arg org.opencontainers.image.vendor "$(VENDOR)" \
+		--arg org.opencontainers.image.ref.name "$(DISTVERSION)" \
+		--arg org.opencontainers.image.version "$(DISTVERSION)" \
+		--arg org.opencontainers.image.url "$(URL)" \
+		'$$ARGS.named | with_entries(select(.value |. !=null and . != ""))' > $@
+
+# Use jq to create the OCI image configuration.
+$(pkg)_config.json:
+	@jq -n \
+		--arg os "$(PORTNAME)" \
+		--arg os.version "$(pkg_os_ver)" \
+		--arg architecture "$(pkg_arch)" \
+		--arg created "$$(date +%Y-%m-%dT%TZ)" \
+		'$$ARGS.named | with_entries(select(.value |. !=null and . != ""))' > $@
+
+# Use jq to create the OCI image manifest annotations.
+$(pkg)_annotations.json: $(pkg).trunk $(pkg)_config.json
+	@anno=$$(jq -n \
+	    --arg org.opencontainers.image.created "$$(date +%Y-%m-%dT%TZ)" \
+		--arg org.opencontainers.image.title "$(pkg).trunk" \
+		--arg org.opencontainers.image.licenses "$(LICENSE)" \
+		--arg org.opencontainers.image.description "$(DESCRIPTION)" \
+		--arg org.opencontainers.image.source "$(REPO_URL)" \
+		--arg org.opencontainers.image.vendor "$(VENDOR)" \
+		--arg org.opencontainers.image.ref.name "$(DISTVERSION)" \
+		--arg org.opencontainers.image.version "$(DISTVERSION)" \
+		--arg org.opencontainers.image.url "$(URL)" \
+		--arg org.pgxn.trunk.pg.version "$(VERSION)" \
+		--arg org.pgxn.trunk.pg.major "$(MAJORVERSION)" \
+		--arg org.pgxn.trunk.pg.version_num "$(VERSION_NUM)" \
+		--arg org.pgxn.trunk.version "0.1.0" \
+		'$$ARGS.named | with_entries(select(.value |. !=null and . != ""))' \
+	) && jq -n \
+		--argjson '$$manifest' "$$anno" \
+		'$$ARGS.named' > $@
 
 $(pkg).trunk: package
 	tar zcvf $@ $(pkg_dir)