Merge pull request #132 from heroku/automate-releases

Adding tools for converting docker stack to heroku stack in travis pipeline.

.travis.yml

@@ -3,11 +3,6 @@ dist: trusty
 language: shell
 services:
 - docker
-branches:
-  # /!\ Every build will always trigger a deploy
-  only:
-  - master
-  - /^v\d+(\.\d+)?(\.\d+)?(-\S*)?$/
 env:
   matrix:
   - STACK=cedar-14 IMAGE_TAG=heroku/cedar:14 PRIVATE_IMAGE_TAG=heroku/cedar-private:14
@@ -16,16 +11,29 @@ env:
   global:
   - secure: qsOv8gjVqCeV7kqx8FntMKD51EZCO2j7oTY8fWwqgndRCT5kHtcJA0vGMW0rjGDlDMuEB5wyUc2RT9ZWMslyKWxgS4Su5zSdAlE8D6TXXpHDU1ZPmOA+cnNm4TIMIrcUlGeCWcK4NNe8Vg7VvrTstooLB/ZjgaCKylkxs7zWq2g= # DOCKER_HUB_USERNAME
   - secure: Dlf4eHJVTQ1bCZuLaaNoKjUmS8SL7Jn2BZCdP7UHZhte+RKchCjajhoObT/fAZ3c5EFfV2pd8NKkWPW0REwN/lodgRPeslYiyNaiIRCVoKRGASxHXHO17wwkMNor2Bi7u6InL3WC1t5YbY2abVaeavtz1lAyn1XTNysc+/s1onw= # DOCKER_HUB_PASSWORD
+  - secure: qTImKa1qsHdMmreZHEOU1JgY2adQ5E/ZHIJw3Nvm5tDqz5lbG5dUahIj1JvzP+FdIoEo6R+0t8C8xD9mBwnz4nMmnahQMnYzYltC9bO9kA7cSI/dktAjQKSBLwtDacFB6jT0UVf7lE7Cxxy70q8L5lkvbqgCxDvU7C16TinYgKA= # SIGNING_KEY_PASSWORD
 script:
-- "bin/build.sh $STACK $IMAGE_TAG $IMAGE_TAG-build"
+- bin/build.sh $STACK $IMAGE_TAG $IMAGE_TAG-build
 - |
   status="$(git status --porcelain)"
   if [[ -n "$status" ]]; then
     echo -e "Regenerated files differ from checked-in versions!\n${status}\n"
     git diff --exit-code
   fi
 deploy:
-  provider: script
-  script: sh bin/publish.sh
-  on:
-    all_branches: true
+  - provider: script
+    script: bin/convert-and-publish-to-heroku.sh
+    on:
+      tags: true
+  - provider: script
+    script: sh bin/publish-to-dockerhub.sh
+    on:
+      all_branches: true
+before_install:
+- |
+  # decrypt private key for signing images
+  openssl aes-256-cbc \
+    -K $encrypted_6cacfc7df997_key \
+    -iv $encrypted_6cacfc7df997_iv \
+    -in ".travis/stack_images.key.enc" \
+    -out /tmp/stack.key -d

BUILD.md

@@ -21,3 +21,17 @@ When building Stack Images for release, we use the Travis build system.
 
 * Any push to master will build the images and push the `nightly` tag.
 * Any new tag will build the image and push the `latest` tag, as well as one with the name of the GIT tag.
+
+# Releasing Stack Images Locally (Prime)
+
+When building Stack Images for relase locally, you'll need a number of additional steps.
+
+    # Build the stack image(s) as you would above
+    cd stack-images/tools
+    # build the stack-image-tooling
+    docker build . -t heroku/stack-image-tools
+    # SET MANIFEST_APP_URL and MANIFEST_APP_TOKEN values, this is the app that controls the bucket for images and metadata about the images (Cheverny)
+    # You must have a private key (tmp/stack.key) to sign the images with (and a password on that key at $STACK_KEY_PASSWORD)
+    docker run -it --rm --privileged -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/stack.key:/tmp/stack.key -e "SIGNING_KEY_PASSWORD=$STACK_KEY_PASSWORD" -e "MANIFEST_APP_URL=$MANIFEST_APP_URL" -e "MANIFEST_APP_TOKEN=$MANIFEST_APP_TOKEN" heroku/stack-image-tools STACK
+    # this will use your local docker image and convert it to a heroku stack image
+    # it will then upload this image and the staging manifest via the MANIFEST_APP

README.md

@@ -5,7 +5,7 @@
 This repository holds recipes for building [Heroku stack images](https://devcenter.heroku.com/articles/stack).  The recipes are also rendered into Docker images that are available on Docker Hub:
 
 * [Cedar-14 Docker image](https://registry.hub.docker.com/u/heroku/cedar/)
-* [Heroku-16 Docker image](https://registry.hub.docker.com/u/heroku/heroku/)
+* [Heroku-16/Heroku-18 Docker image](https://registry.hub.docker.com/u/heroku/heroku/)
 
 ### Learn more
 

bin/convert-and-publish-to-heroku.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+sudo cp tools/bin/* /usr/local/bin
+sudo convert-to-heroku-stack-image $STACK

tools/Dockerfile

@@ -0,0 +1,11 @@
+FROM ubuntu:16.04
+
+RUN apt-get update
+RUN apt-get install docker.io -y
+RUN apt-get install jq -y
+RUN apt-get install curl -y
+
+COPY bin /usr/local/bin
+
+VOLUME ["/var/run/docker.sock"]
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint"]

tools/bin/abort

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -e
+
+echo "$@"
+exit 1

tools/bin/capture-docker-stack

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+[ $# -ge 1 ] || abort usage: $(basename $0) STACK [VERSION]
+[ $UID = 0 ] || abort fatal: must be called with sudo
+
+STACK=$1
+STACK_NAME=$(echo $STACK | cut -d '-' -f 1)
+STACK_VERSION=$(echo $STACK | cut -d '-' -f 2-)
+
+DOCKER_IMAGE=heroku/$STACK_NAME:$STACK_VERSION
+DOCKER_IMAGE_VERSION=$(docker inspect $DOCKER_IMAGE | jq .[].Id | cut -d ':' -f 2 | cut -b 1-12)
+
+IMG_BASE=${STACK_NAME}64-$STACK_VERSION-$DOCKER_IMAGE_VERSION
+IMG=/tmp/$IMG_BASE.img
+IMG_MNT=/tmp/$IMG_BASE
+IMG_GZ=/tmp/$IMG_BASE.img.gz
+IMG_SHA256=/tmp/$IMG_BASE.img.sha256
+IMG_MANIFEST=/tmp/$IMG_BASE.manifest
+SIG=/tmp/$IMG_BASE.img.signature
+PRIVATE_KEY=/tmp/stack.key
+IMG_PKG_VERSIONS=/tmp/$IMG_BASE.pkg.versions
+
+display Starting capture for $STACK $DOCKER_IMAGE_VERSION at $(date)
+
+display Creating image file $IMG
+make-filesystem-image $IMG |& indent
+
+display Mounting image $IMG_MNT
+mount-filesystem-image $IMG $IMG_MNT |& indent
+
+display Copying stack to image
+export-docker-image $DOCKER_IMAGE $IMG_MNT |& indent
+
+display Modifying image directories and files
+install-heroku-files $IMG_MNT |& indent
+
+display Unmounting image
+umount $IMG_MNT |& indent
+
+display Signing image
+sign-image $IMG $SIG $PRIVATE_KEY
+
+display SHA256ing and gzipping image
+make-image-archive $IMG $IMG_SHA256 |& indent
+cat $IMG_SHA256
+
+if update-manifest; then
+    display Starting push at $(date)
+    display Capture Package Versions
+    capture-package-versions $DOCKER_IMAGE $IMG_PKG_VERSIONS
+    display Uploading files
+    upload-image $IMG_GZ $IMG_SHA256 $IMG_MANIFEST $STACK $DOCKER_IMAGE_VERSION $SIG $IMG_PKG_VERSIONS |& indent
+    display Updating staging manifest
+    publish-manifests
+else
+    display Skipping push and manifest update
+fi
+
+display Finished capture for $STACK $DOCKER_IMAGE_VERSION

tools/bin/capture-package-versions

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+DOCKER_IMAGE=$1
+OUTPUT_FILE=$2
+
+docker run --rm -ti heroku/heroku:16 dpkg-query -W -f "\${package},\${version}\n" | jq -s -R -f /usr/local/bin/csv-to-array.jq > $OUTPUT_FILE

tools/bin/convert-to-heroku-stack-image

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e -o pipefail
+VERSION_PREFIX=$(date '+%Y%m%d-%H%M%S')
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    cedar-14)
+      capture-docker-stack "$1"       "$VERSION_PREFIX"
+      ;;
+    *)
+      capture-docker-stack "$1"       "$VERSION_PREFIX"
+      capture-docker-stack "$1-build" "$VERSION_PREFIX"
+      ;;
+  esac
+  shift
+done

tools/bin/csv-to-array.jq

@@ -0,0 +1,13 @@
+[
+  [
+    split("\r\n")[]                  # transform csv input into array
+  | split(",")                    # where first element has key names
+  | select(length==2)              # and other elements have values
+  ]
+  | {h:.[0], v:.[1:][]}            # {h:[keys], v:[values]}
+  | [.h, .v]   # [ [keys], [values] ]
+  | [ transpose[]                  # [ [key,value], [key,value], ... ]
+      | {key:.[0], value:.[1]}     # [ {"key":key, "value":value}, ... ]
+    ]
+  | from_entries                   # { key:value, key:value, ... }
+]

tools/bin/display

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -e
+
+exec echo $'\n----->' "$@"

tools/bin/docker-entrypoint

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+set -e -o pipefail
+convert-to-heroku-stack-image "$@"

tools/bin/export-docker-image

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+DOCKER_IMAGE="$1"
+MNT="$2"
+
+CONTAINER="$(docker create "$DOCKER_IMAGE")"
+trap 'docker rm "$CONTAINER" > /dev/null' EXIT
+
+docker export "$CONTAINER" | tar -x -C "$MNT" --exclude=lib/modules bin etc lib lib64 sbin usr var/lib

tools/bin/indent

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -e
+
+exec sed -u "s/^/       /"

tools/bin/install-heroku-files

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -e
+
+IMG_MNT="$1"
+
+# etc/heroku is a mountpoint for runtime dyno information
+# sys is there so users of the stack can mount a sysfs on it (/sys is the default location)
+for d in app tmp proc sys dev var var/log var/tmp etc/heroku; do
+  mkdir -p "$IMG_MNT/$d"
+done
+
+echo "127.0.0.1 localhost localhost.localdomain" > "$IMG_MNT/etc/hosts"
+
+echo "heroku-runtime" > "$IMG_MNT/etc/hostname"
+
+for f in etc/profile etc/bash.bashrc; do
+  echo "export PS1='\\[\\033[01;34m\\]\\w\\[\\033[00m\\] \\[\\033[01;32m\\]$ \\[\\033[00m\\]'" > "$IMG_MNT/$f"
+done
+
+cat >"$IMG_MNT/etc/resolv.conf" <<'EOF'
+nameserver 172.16.0.23
+domain z-2.compute-1.internal
+search z-2.compute-1.internal
+EOF

tools/bin/make-filesystem-image

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+IMG="$1"
+
+mkdir -p "$(dirname "$IMG")"
+dd if=/dev/zero of="$IMG" bs=100M count=24
+yes | mkfs -t ext3 -m 1 "$IMG"
+tune2fs -c 0 -i 0 "$IMG"

tools/bin/make-image-archive

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+IMG="$1"
+IMG_SHA256="$2"
+
+sha256sum "$IMG" | cut -d ' ' -f 1 | tr -d '\n' > "$IMG_SHA256"
+gzip "$IMG"

tools/bin/mount-filesystem-image

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+IMG="$1"
+IMG_MNT="$2"
+
+mkdir -p "$IMG_MNT"
+mount -o loop,noatime,nodiratime "$IMG" "$IMG_MNT"

tools/bin/publish-manifests

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+
+if ls /tmp/*64-*.manifest >& /dev/null; then
+    jq --null-input \
+        '[inputs]
+        | map({
+            name: .name,
+            id: .id,
+            sha256: .sha256,
+            signature: .signature
+        }) | {stacks: .}' \
+        /tmp/*64-*.manifest |
+        curl \
+            --fail --user-agent 'Stack Image Tools' \
+            --header "Authorization: Bearer $MANIFEST_APP_TOKEN" \
+            --header "Content-Type: application/json" \
+            --request PATCH \
+            --data @- \
+            "$MANIFEST_APP_URL/manifest/staging"
+fi

tools/bin/sign-image

@@ -0,0 +1,32 @@
+#!/bin/bash
+# This script will create a digital signature for a stack image. The signature is written to
+# a temporary file (base64 encoded) so that it is accessable to other scripts. It uses the
+# Usage: sign-image <image file name> <signature file name> <private key file>
+# Preconditions: The environment variable SIGNING_KEY_PASSWORD should be defined and contain the passphrase
+# for the private key.
+
+if [ "$#" -ne 3 ]; then
+
+    if [ ! -e "$1" ]; then
+        echo "Image file name parameter (1) is missing."
+    fi
+
+    if [ ! -e "$2" ]; then
+        echo "Signature file name parameter (2) is missing."
+    fi
+
+    if [ ! -e "$3" ]; then
+        echo "Private key file parameter (3) is missing."
+    fi
+
+    echo "Usage: sign-image <image file name> <signature file name> <private key file> "
+    exit 1
+fi
+
+# Create digital signature
+binary_sig_file=$(mktemp /tmp/sig.XXXXXX)
+trap "{ rm -f $binary_sig_file; }" EXIT
+
+openssl dgst -sha256 -sign "$3" -passin env:SIGNING_KEY_PASSWORD "$1" > "$binary_sig_file"
+# Base 64 encode the signature and write encoded signature to a file
+openssl enc -a -e -out "$2" -in "$binary_sig_file"

tools/bin/update-manifest

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+[ -z "$MANIFEST_APP_URL" ] && echo "Missing manifest app url" && exit 1
+[ -z "$MANIFEST_APP_TOKEN" ] && echo "Missing manifest app token" && exit 1
+
+exit 0