[ADP-3400] Automate part of the release process as CI steps (#4749)

paolino · web-flow · commit 5ba3cbc3b2b8 · 2024-08-27T16:13:30.000Z
This PR remove some manual steps from the release process. In particular - create release tag - create release - upload artifacts - upload the titles and links of the PRs since last release, for further human processing Because the release process is rare, this PR introduce a running draft release called nightly, which runs the automated steps on a running tag that gets rewritten every day. More work should be done to - collect automatically the dockerhub image link. - collect the bump.sh to the current change ADP-3400
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -510,11 +510,12 @@ steps:
       if: |
         build.branch !~ /^gh-readonly-queue\/master/
           && build.branch != "master"
-          && ( build.env("RELEASE_CANDIDATE") == null || build.env("TEST_RC") == "TRUE" )
+          && build.env("RELEASE_CANDIDATE") == null
       key: block-macos
 
     - label: Build Integration Tests (macOS, arm64)
       key: macos-arm64-tests-build-integration
+      if: build.env("TEST_RC") == null || build.env("TEST_RC") == "FALSE"
       depends_on: block-macos
       command: nix build -L .#packages.aarch64-darwin.integration-exe
       agents:
@@ -610,10 +611,8 @@ steps:
       if: |
         build.branch !~ /^gh-readonly-queue\/master/
           && build.branch != "master"
-          && (build.env("RELEASE_CANDIDATE") == null
-              || build.env("TEST_RC") == "TRUE"
-              )
           && build.branch != "rc-latest"
+          && build.env("RELEASE_CANDIDATE") == null
       key: trigger-build-windows-artifacts
 
     - label: Build Package (windows)
@@ -627,6 +626,7 @@ steps:
 
     - label: Build Testing Bundle (windows)
       key: windows-testing-bundle
+      if: build.env("TEST_RC") == null || build.env("TEST_RC") == "FALSE"
       depends_on:
         - trigger-build-windows-artifacts
       command: nix build -o result/windows-tests .#ci.artifacts.win64.tests
@@ -635,7 +635,7 @@ steps:
         system: ${linux}
 
     - label: Tag as Release Candidate (windows)
-      if: build.env("RELEASE_CANDIDATE") != null
+      if: build.env("RELEASE_CANDIDATE") == null || build.env("TEST_RC") == "FALSE"
       depends_on:
         - windows-package
         - windows-testing-bundle
@@ -650,21 +650,20 @@ steps:
       - linux-artifacts
     steps:
 
-      - block: Docker Build
-        depends_on: []
-        if: build.env("TEST_RC") == "TRUE"
-        key: docker-build-block
-
       - label: Build Docker Image
         key: docker-build
-        depends_on: docker-build-block
         commands:
           ./scripts/buildkite/main/docker-build.sh
         artifact_paths:
           - ./artifacts/*.tgz
         agents:
           system: x86_64-linux
 
+      - block: Docker Build
+        depends_on: []
+        if: build.env("TEST_RC") == "TRUE"
+        key: docker-build-block
+
       - label: Push Docker Image
         depends_on:
           - docker-build
diff --git a/.buildkite/release.yml b/.buildkite/release.yml
@@ -24,22 +24,63 @@ steps:
     commands:
       - ./scripts/buildkite/release/generate-trigger.sh | buildkite-agent pipeline upload
 
-  - label: Push swagger daily
-    depends_on: main-pipeline-build
+  - label: Collect changes
+    key: collect-changes
+    depends_on: add-release-commits
     commands:
-      - nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-to-bump.sh
-    artifacts:
-      - artifacts/api-diffs.md
-    env:
-      RELEASE: false
+      - nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/make_changelog.sh
     agents:
       system: x86_64-linux
+    artifacts:
+      - artifacts/changes.md
+
+  - group : Nightly
+    key: nightly
+    depends_on: main-pipeline-build
+    steps:
+      - label: Push swagger nightly
+        commands:
+          - nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-to-bump.sh
+        artifacts:
+          - artifacts/api-diffs.md
+        env:
+          RELEASE: false
+        agents:
+          system: x86_64-linux
+
+      - label: Push nightly release tag
+        key: push-nightly-tag
+        commands:
+          - nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-tag.sh
+        agents:
+          system: x86_64-linux
+        env:
+          RELEASE: false
+
+      - label: Push nightly release
+        depends_on: push-nightly-tag
+        key: push-nightly-release
+        commands:
+          - nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-release.sh
+        agents:
+          system: x86_64-linux
+        env:
+          RELEASE: false
+
+      - label: Push nightly release artifacts
+        depends_on: push-nightly-release
+        commands:
+          - nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-artifacts.sh
+        agents:
+          system: x86_64-linux
+        env:
+          RELEASE: false
 
   - group: Release
-    if: build.branch =~ /^release-candidate/
+    depends_on: nightly
+    if: build.branch == "master"
     steps:
       - block: Create a release
-        depends_on: main-pipeline-build
         key: create-release
 
       - label: Push swagger release
@@ -51,4 +92,33 @@ steps:
         env:
           RELEASE: true
         agents:
-          system: x86_64-linux
+          system: x86_64-linux
+
+      - label: Push release tag
+        key: push-release-tag
+        depends_on: create-release
+        commands:
+          - nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-tag.sh
+        agents:
+          system: x86_64-linux
+        env:
+          RELEASE: true
+
+      - label: Push release
+        depends_on: push-release-tag
+        key: push-release
+        commands:
+          - nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-release.sh
+        agents:
+          system: x86_64-linux
+        env:
+          RELEASE: true
+
+      - label: Push release artifacts
+        depends_on: push-release
+        commands:
+          - nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-artifacts.sh
+        agents:
+          system: x86_64-linux
+        env:
+          RELEASE: true
diff --git a/scripts/buildkite/release/flake.nix b/scripts/buildkite/release/flake.nix
@@ -32,6 +32,8 @@
             pkgs.gnutar
             pkgs.gnupg
             pkgs.nodejs_22
+            pkgs.gh
+            pkgs.gettext
           ];
           shellHook = ''
             # use this hook to set up additional environment variables
diff --git a/scripts/buildkite/release/generate-trigger.sh b/scripts/buildkite/release/generate-trigger.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#! /usr/bin/env bash
 
 set -euo pipefail
 
@@ -7,6 +7,13 @@ VERSION=$(buildkite-agent meta-data get "release-version")
 BRANCH=$(buildkite-agent meta-data get "release-candidate-branch")
 CABAL=$(buildkite-agent meta-data get "release-cabal-version")
 TEST_RC=$(buildkite-agent meta-data get "test-rc")
+BASE_BUILD=$(buildkite-agent meta-data get "base-build")
+
+if [ "$TEST_RC" == "TRUE" ]; then
+    title="Test Release Candidate of $VERSION"
+else
+    title="Release Candidate of $VERSION"
+fi
 
 cat << YAML
 steps:
@@ -17,7 +24,7 @@ steps:
     build:
         commit: $COMMIT
         branch: $BRANCH
-        message: Release Candidate of $VERSION
+        message: $title
         env:
             RELEASE_CANDIDATE: "$VERSION"
             TEST_RC: "$TEST_RC"
@@ -26,5 +33,6 @@ steps:
             release-candidate-commit: "$COMMIT"
             release-candidate-branch: "$BRANCH"
             release-cabal-version: "$CABAL"
+            triggered-by: "$BASE_BUILD"
 
 YAML
diff --git a/scripts/buildkite/release/make_changelog.sh b/scripts/buildkite/release/make_changelog.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+since_date="$(buildkite-agent meta-data get last-release-date)"
+
+fetch_prs() {
+    curl --silent \
+    -H "Accept: application/vnd.github+json" \
+    -H "Authorization: Bearer $GITHUB_TOKEN" \
+    -H "X-GitHub-Api-Version: 2022-11-28" \
+    "https://api.github.com/search/issues?q=repo:cardano-foundation/cardano-wallet+is:pr+is:merged+merged:%3E$since_date"
+    }
+
+prs=$(fetch_prs | jq '.items | map({date:.pull_request.merged_at | split("T")[0], number:.number,title:.title,labels:.labels| map(.name),author:.user.login })')
+
+items=$(jq 'map("\(.date), #\(.number), \(.author), \(.labels | join(", ")), \(.title)")' <<< "$prs")
+item_max=$(jq 'length - 1' <<< "$items")
+
+mkdir -p artifacts
+
+for i in $(seq 0 "$item_max"); do
+    item=$(jq -r ".[$i]" <<< "$items")
+    echo "$item" \
+        | sed -E 's,\[(ADP\-[0-9]+)\],[\1](https://cardanofoundation.atlassian.net/browse/\1),g' \
+        | sed -E 's,\[ADP-x+\],,g' \
+        | sed -E 's,#([0-9]+),[#\1](https://github.com/cardano-foundation/cardano-wallet/pull/\1),g' \
+        >> artifacts/changes.md
+done
diff --git a/scripts/buildkite/release/push-artifacts.sh b/scripts/buildkite/release/push-artifacts.sh
@@ -0,0 +1,38 @@
+#! /usr/bin/env bash
+
+set -euox pipefail
+
+base_build=$(buildkite-agent meta-data get base-build)
+NEW_GIT_TAG=$(buildkite-agent meta-data get release-version)
+
+if [ "$RELEASE" == "false" ]; then
+    TAG=nightly
+else
+    TAG=$NEW_GIT_TAG
+fi
+
+main_build=$(curl -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
+    -X GET "https://api.buildkite.com/v2/builds" \
+    | jq ".[] | select(.meta_data.\"triggered-by\" == \"$base_build\")" \
+    | jq .number)
+
+mkdir -p artifacts
+
+artifact() {
+    local artifact_name=$1
+    # shellcheck disable=SC2155
+    local artifact_value=$(curl -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
+        -X GET "https://api.buildkite.com/v2/organizations/cardano-foundation/pipelines/cardano-wallet/builds/$main_build/artifacts" \
+        | jq -r ".[] | select(.filename == \"$artifact_name\") \
+        | .download_url")
+    curl -H "Authorization: Bearer $BUILDKITE_API_TOKEN" -L \
+        -o "artifacts/$artifact_name" \
+        "$artifact_value"
+    gh release upload "$TAG" "artifacts/$artifact_name"
+}
+
+artifact "cardano-wallet-$NEW_GIT_TAG-linux64.tar.gz"
+artifact "cardano-wallet.exe-$NEW_GIT_TAG-win64.zip"
+artifact "cardano-wallet-$NEW_GIT_TAG-macos-silicon.tar.gz"
+artifact "cardano-wallet-$NEW_GIT_TAG-macos-intel.tar.gz"
+artifact "cardano-wallet-$NEW_GIT_TAG-docker-image.tgz"
diff --git a/scripts/buildkite/release/push-release.sh b/scripts/buildkite/release/push-release.sh
@@ -0,0 +1,41 @@
+#! /usr/bin/env bash
+
+set -euox pipefail
+
+NEW_GIT_TAG=$(buildkite-agent meta-data get release-version)
+
+if [ "$RELEASE" == "false" ]; then
+    TAG=nightly
+    title="Nightly $NEW_GIT_TAG"
+else
+    TAG=$NEW_GIT_TAG
+    title="Release $TAG"
+fi
+
+buildkite-agent artifact download artifacts/changes.md .
+# shellcheck disable=SC2155
+export CHANGES=$(cat artifacts/changes.md)
+
+# shellcheck disable=SC2155
+export NODE_TAG=$(buildkite-agent meta-data get node-tag)
+
+# shellcheck disable=SC2034
+export BUMP_CHANGES=xxx
+
+# shellcheck disable=SC2034
+export DOCKER_SHA=xxx
+
+# shellcheck disable=SC2016
+envsubst \
+    < "$RELEASE_SCRIPTS_DIR/release-template.md" \
+    > "$RELEASE_SCRIPTS_DIR/release-template-final.md"
+
+if [ "$RELEASE" == "false" ]; then
+    gh release delete nightly --yes || true
+fi
+
+gh release create \
+    -d \
+    -F "$RELEASE_SCRIPTS_DIR/release-template-final.md" \
+    -t "$title" \
+    "$TAG"
diff --git a/scripts/buildkite/release/push-tag.sh b/scripts/buildkite/release/push-tag.sh
@@ -0,0 +1,26 @@
+#! /usr/bin/env bash
+
+set -euox pipefail
+
+RELEASE_GIT_COMMIT=$(buildkite-agent meta-data get release-candidate-commit)
+
+git tag -l | xargs git tag -d
+git fetch --tags
+
+if [ "$RELEASE" == "false" ]; then
+    git tag -d nightly || true
+    TAG=nightly
+else
+    TAG=$(buildkite-agent meta-data get release-version)
+    exists=$(git tag -l "$TAG")
+    if [ -n "$exists" ]; then
+        echo "Tag $TAG already exists. Remove it before proceeding."
+        exit 1
+    fi
+fi
+
+git tag -m "$TAG" "$TAG" "$RELEASE_GIT_COMMIT"
+
+git remote set-url origin "https://$GITHUB_TOKEN@github.com/cardano-foundation/cardano-wallet.git"
+
+git push origin -f "$TAG"
diff --git a/scripts/buildkite/release/push-to-bump.sh b/scripts/buildkite/release/push-to-bump.sh
@@ -1,4 +1,4 @@
-#! /bin/bash
+#! /usr/bin/env bash
 
 set -euox pipefail
 
diff --git a/scripts/buildkite/release/release-candidate.sh b/scripts/buildkite/release/release-candidate.sh
@@ -14,7 +14,8 @@ tag_cabal_ver() {
   tag_date "$1" | sed -e s/-0/-/g -e s/-/./g
 }
 
-git fetch --tags --force
+git tag -l | xargs git tag -d
+git fetch --tags
 
 BASE_COMMIT=$(git rev-parse HEAD)
 
@@ -28,6 +29,8 @@ NEW_CABAL_VERSION=$(tag_cabal_ver "$NEW_GIT_TAG")
 
 OLD_GIT_TAG=$( git tag -l "v2*-*-*" | sort | tail -n1)
 
+LAST_RELEASE_DATE=$(tag_date "$OLD_GIT_TAG")
+
 OLD_CABAL_VERSION=$(tag_cabal_ver "$OLD_GIT_TAG")
 
 CARDANO_NODE_TAG=$(cardano-node version | head -n1 | awk '{print $2}')
@@ -72,3 +75,6 @@ buildkite-agent meta-data set "release-candidate-commit" "$RELEASE_COMMIT"
 buildkite-agent meta-data set "release-candidate-branch" "$RELEASE_CANDIDATE_BRANCH"
 buildkite-agent meta-data set "release-cabal-version" "$NEW_CABAL_VERSION"
 buildkite-agent meta-data set "test-rc" "$TEST_RC"
+buildkite-agent meta-data set "base-build" "$BUILDKITE_BUILD_ID"
+buildkite-agent meta-data set "node-tag" "$CARDANO_NODE_TAG"
+buildkite-agent meta-data set "last-release-date" "$LAST_RELEASE_DATE"
diff --git a/scripts/buildkite/release/release-template.md b/scripts/buildkite/release/release-template.md
diff --git a/scripts/make_changelog.sh b/scripts/make_changelog.sh

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-#! /bin/bash`
	`1`	`+#! /usr/bin/env bash`
`2`	`2`
`3`	`3`	`set -euox pipefail`
`4`	`4`