Skip to content

Commit

Permalink
ci: add script and pipeline for rebuilding images [skip ci] [DEVOPS-2…
Browse files Browse the repository at this point in the history
…94] (#15611)

* ci: add script and pipeline for rebuilding images [skip ci]

* ci: build images with Jib CLI build file [skip ci]

* ci: add option to rebuild all supported images in pipeline [skip ci]

* ci: create manifest lists for rolling tags [skip ci]

* ci: explicitly delete war file and unarchived dir [skip ci]

* ci: update build docker image script for dev images [skip ci]

* ci: use build-docker-image script in stable pipeline [skip ci]

* ci: use build-docker-image script in dev pipeline [skip ci]

* docs: mention immutable and rolling tags in DOCKERHUB description [skip ci]

* chore: update stable versions json source [skip ci]

---------

Co-authored-by: Philip-Larsen-Donnelly <[email protected]>
  • Loading branch information
radnov and Philip-Larsen-Donnelly authored Dec 4, 2023
1 parent b890cb8 commit c31321e
Show file tree
Hide file tree
Showing 6 changed files with 365 additions and 23 deletions.
218 changes: 218 additions & 0 deletions dhis-2/build-docker-image.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,218 @@
#!/usr/bin/env bash

set -euo pipefail

IMAGE_REPOSITORY=${IMAGE_REPOSITORY:-'dhis2/core'}
IMAGE_APP_ROOT=${IMAGE_APP_ROOT:-'/usr/local/tomcat/webapps/ROOT'}
IMAGE_USER=${IMAGE_USER:-'65534'}
WAR_PATH=${WAR_PATH:-'dhis-2/dhis-web/dhis-web-portal/target/dhis.war'}
UNARCHIVED_WAR_DIR=${UNARCHIVED_WAR_DIR:-'dhis2-war'}
JIB_BUILD_FILE=${JIB_BUILD_FILE:-'jib.yaml'}

downloaded_war_name='downloaded-dhis2.war'
old_version_schema_prefix='2'
stable_versions_json="$(curl -fsSL "https://releases.dhis2.org/v1/versions/stable.json")"

function help() {
echo 'Available options:'
echo '-t <tag> DHIS2 version tag to build.'
echo '-d Create image from a "dev" version.'
echo '-r Rebuild image with an existing WAR for the given DHIS2 version. Without this option the image will be built with a new WAR.'
echo '-h Print this help message.'
echo
echo 'Example: build-docker-image.sh -t 40.1.1 -r'
}

function create_immutable_tag() {
current_date="$(date -u +'%Y%m%dT%H%M%SZ')" # ISO 8601 format

main_image_tag="$image_tag-$current_date"
}

function create_rolling_tags() {
IFS=. read -ra image_tag_segments <<< "$image_tag"

major="${image_tag_segments[0]}"
minor="${image_tag_segments[1]}"
patch="${image_tag_segments[2]}"

# Always create M.m.p and 2.M.m.p
rolling_tags=(
"$image_tag"
"$old_version_schema_prefix.$image_tag"
)

# If patch(hotfix) is the latest for given minor(patch) create M.m + 2.M.m
latest_hotfix_version="$(
echo "$stable_versions_json" |
jq -r --argjson major "$major" --argjson minor "$minor" \
'.versions[] | select(.version == $major) .patchVersions[] | select(.version == $minor) .hotfixVersion' |
sort -n |
tail -1
)"
if [[ "$patch" == "$latest_hotfix_version" ]]; then
echo "The patch version $patch is the latest for the minor version $minor"
rolling_tags+=(
"$major.$minor"
"$old_version_schema_prefix.$major.$minor"
)
fi

# If version is the latest create M + 2.M
latest_version="$(
echo "$stable_versions_json" |
jq -r --argjson major "$major" \
'.versions[] | select(.version == $major) | "\(.version).\(.latestPatchVersion).\(.latestHotfixVersion)"'
)"
if [[ "$image_tag" == "$latest_version" ]]; then
echo "Version $image_tag is the latest version for the supported major version $major"
rolling_tags+=(
"$major"
"$old_version_schema_prefix.$major"
)
fi
}

function list_tags() {
echo 'Immutable tag:'
echo "$main_image_tag"

echo 'Rolling tags:'
echo "${rolling_tags[@]}"
}

function use_existing_war() {
echo 'Image will be rebuilt with existing WAR'

war_url="$(
echo "$stable_versions_json" |
jq -r --arg image_tag "$image_tag" '.versions[] | select(.supported == true) .patchVersions[] | select(.displayName == $image_tag) .url'
)"
echo "Downloading from $war_url ..."
rm -rf "$downloaded_war_name"
curl -o "$downloaded_war_name" "$war_url"

known_war_sha256="$(
echo "$stable_versions_json" |
jq -r --arg image_tag "$image_tag" '.versions[] | select(.supported == true) .patchVersions[] | select(.displayName == $image_tag) .sha256'
)"

sha256sum_output="$(sha256sum "$downloaded_war_name")"
downloaded_war_sha256="${sha256sum_output%% *}" # strip file name from sha256sum output with variable expansion

if [[ "$downloaded_war_sha256" != "$known_war_sha256" ]]; then
echo "Downloaded WAR sha256 sum ($downloaded_war_sha256) doesn't match known sha256 sum ($known_war_sha256)!"
exit 1
fi

echo "Unarchiving WAR to $UNARCHIVED_WAR_DIR ..."
rm -rf "./$UNARCHIVED_WAR_DIR"
unzip -q -o "$downloaded_war_name" -d "./$UNARCHIVED_WAR_DIR"
}

function use_new_war() {
echo "Image will be built with new WAR from $WAR_PATH"
rm -rf "./$UNARCHIVED_WAR_DIR"
unzip -q -o "$WAR_PATH" -d "./$UNARCHIVED_WAR_DIR"
}

function build_main_image() {
echo "Building image for version $image_tag, based on $BASE_IMAGE ..."

jib build \
--build-file "$JIB_BUILD_FILE" \
--target "$IMAGE_REPOSITORY:$main_image_tag" \
--parameter unarchivedWarDir="$UNARCHIVED_WAR_DIR" \
--parameter imageAppRoot="$IMAGE_APP_ROOT" \
--parameter baseImage="$BASE_IMAGE" \
--parameter imageUser="$IMAGE_USER" \
--parameter gitCommit="$GIT_COMMIT" \
--parameter gitBranch="$GIT_BRANCH" \
--parameter dhis2Version="$DHIS2_VERSION" \
--parameter timestamp="$(date +'%s000')" # Unix time with zeroed milliseconds; will be shown as "2023-11-02T14:40:32Z"
}

# To create additional rolling tags for a multi-architecture image (manifest list) we have to create a new manifest list,
# based on the supported image architectures' sha256 digests.
function create_additional_manifests() {
echo "Pulling $IMAGE_REPOSITORY:$main_image_tag for tagging ..."
docker image pull "$IMAGE_REPOSITORY:$main_image_tag"

for tag in "${rolling_tags[@]}"; do
# shellcheck disable=SC2207
manifest_digests=($(docker manifest inspect "$IMAGE_REPOSITORY:$main_image_tag" | jq -r '.manifests[] .digest'))

# shellcheck disable=SC2145
echo "Creating new manifest with digests ${manifest_digests[@]} ..."

# No double quotes for the $manifest_digests, as we want them treated as separate elements and prefixed with the repository.
# shellcheck disable=SC2068
docker manifest create "$IMAGE_REPOSITORY:$tag" ${manifest_digests[@]/#/ $IMAGE_REPOSITORY@}

echo "Pushing new manifest $IMAGE_REPOSITORY:$tag ..."
docker manifest push "$IMAGE_REPOSITORY:$tag"
done
}

while getopts 'ht:dr' option; do
case $option in
h)
help
exit;;
t)
image_tag=$OPTARG;;
d)
dev_image=1;;
r)
rebuild_image=1;;
\?)
echo 'Error: Invalid option'
exit;;
esac
done

if [[ ${dev_image:-} -eq 1 ]]; then
echo 'Creating image from a dev version ...'

# We're not creating an immutable tag with a timestamp for dev images.
main_image_tag="$image_tag"

use_new_war

build_main_image

if [[ "$image_tag" =~ -rc$ ]]; then
echo 'Version is Release Candidate, creating extra old version format tag.'
rolling_tags=("$old_version_schema_prefix.$main_image_tag")
create_additional_manifests
fi

echo 'Done with building dev image. Exiting.'
echo
exit 0
fi

create_immutable_tag

create_rolling_tags

list_tags

if [[ ${rebuild_image:-} -eq 1 ]]; then
use_existing_war
else
use_new_war
fi

jdk_version="$(
echo "$stable_versions_json" |
jq -r --argjson major "$major" '.versions[] | select(.version == $major) .jdk'
)"
BASE_IMAGE="${BASE_IMAGE:-"tomcat:9.0-jre$jdk_version"}"

build_main_image

create_additional_manifests

echo "Done with building $image_tag."
echo
6 changes: 4 additions & 2 deletions docker/DOCKERHUB.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,10 @@
# Supported tags

* [`dhis2/core`](https://hub.docker.com/r/dhis2/core) - images of the release and release-candidate
DHIS2 versions. These images represent the **stable** DHIS2 versions, meaning they won't be
rebuilt in the future.
DHIS2 versions. These images represent the **stable** DHIS2 versions and have immutable tags
like `40.1.0-20231109T170534Z` that are never rebuilt, as well as rolling tags
like `40.1.0`, `2.40.1.0`, `40.1`, `2.40.1`, `40` and `2.40` that point
to the latest immutable tag for the respective version and are regularly rebuilt.

* [`dhis2/core-dev`](https://hub.docker.com/r/dhis2/core-dev) - images of _the latest development_
DHIS2 versions - branches `master` (tagged as `latest`) and the previous 3 supported major
Expand Down
12 changes: 7 additions & 5 deletions jenkinsfiles/dev
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,12 @@ pipeline {
}
}

stage('Publish images') {
stage('Publish image') {
environment {
BASE_IMAGE = "$DOCKER_IMAGE_NAME_PUBLISH_SOURCE"
IMAGE_REPOSITORY = "$DOCKER_IMAGE_NAME"
}

steps {
script {
withDockerRegistry([credentialsId: "docker-hub-credentials", url: ""]) {
Expand All @@ -160,10 +165,7 @@ pipeline {
}

withMaven(options: [artifactsPublisher(disabled: true)]) {
sh "mvn --batch-mode --no-transfer-progress -DskipTests -Dmaven.test.skip=true \
-f dhis-2/dhis-web/dhis-web-portal/pom.xml jib:build -PjibBuild \
-Djib.from.image=${DOCKER_IMAGE_NAME_PUBLISH_SOURCE} -Djib.to.image=${DOCKER_IMAGE_NAME}:${tag} \
-Djib.container.labels=DHIS2_VERSION=${DHIS2_VERSION},DHIS2_BUILD_REVISION=${GIT_COMMIT},DHIS2_BUILD_BRANCH=${env.GIT_BRANCH}"
sh "./dhis-2/build-docker-image.sh -t ${tag} -d"
}
}
}
Expand Down
57 changes: 57 additions & 0 deletions jenkinsfiles/rebuild-image
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
#!/usr/bin/env groovy

@Library('pipeline-library') _

pipeline {
agent {
label 'ec2-jdk11-large-spot'
}

parameters {
booleanParam(name: 'ALL_SUPPORTED_VERSIONS', defaultValue: false, description: '[OPTIONAL] Rebuild for all supported versions.')
string(name: 'VERSIONS', defaultValue: '', description: 'Space separated list of DHIS2 versions to rebuild (new format).')
}

options {
disableConcurrentBuilds()
}

stages {
stage('Rebuild Docker images') {
environment {
OLD_VERSION_SCHEMA_PREFIX = '2'
SUPPORTED_VERSIONS_JSON = sh(returnStdout: true, script: 'curl -fsSL "https://raw.githubusercontent.com/dhis2/dhis2-releases/master/downloads/v1/versions/stable.json" | jq -r \'.versions[] | select(.supported == true)\'').trim()
}

steps {
script {
withDockerRegistry([credentialsId: "docker-hub-credentials", url: ""]) {
env.VERSIONS_TO_REBUILD = env.VERSIONS
if (params.ALL_SUPPORTED_VERSIONS.toBoolean()) {
env.VERSIONS_TO_REBUILD = sh(
returnStdout: true,
script: '''#!/bin/bash
jq -r \'.patchVersions[] | .displayName\' <<< "$SUPPORTED_VERSIONS_JSON" | xargs
''').trim()
}

// We don't have 2.M.m.p tags with 0 patch version in the dhis2-core git repo,
// hence we have to use the version "name" from the stable.json as the matching git tag.
env.VERSIONS_TO_REBUILD.tokenize(" ").each { version ->
sh """#!/bin/bash
export VERSION_NAME=\$(jq -r '.patchVersions[] | select(.displayName == \"$version\") .name' <<< \$SUPPORTED_VERSIONS_JSON)
export DHIS2_VERSION=\$VERSION_NAME
export GIT_BRANCH=\$DHIS2_VERSION
export GIT_COMMIT=\$(git rev-parse \$VERSION_NAME)
echo "DHIS2 version is \$DHIS2_VERSION"
echo "Git commit is \$GIT_COMMIT"
./dhis-2/build-docker-image.sh -t \"$version\" -r
"""
}
}
}
}
}
}
}
Loading

0 comments on commit c31321e

Please sign in to comment.