Update release automation to support Privacy Pro section in release n…

…otes (#2710)

Task/Issue URL: https://app.asana.com/0/1177771139624306/1207157777571909/f

Description:
This change makes it possible to display a separate section in release notes, dedicated to Privacy Pro:
* appcastManager.swift was updated to take pre-formatted HTML document with release notes
* extract_release_notes.sh was updated to generate HTML or Asana rich text release notes, in addition to raw output.
* update_asana_for_release.sh was updated to rely on extract_release_notes.sh for generating
  release notes for the Asana release task.
* workflows were updated to use extract_release_notes.sh to generate HTML documents
* extract_release_notes.sh is now unit-tested using Bats (Bash Automated Testing System).
* PR checks workflow is updated with a job dedicated to running shell scripts unit tests.

.github/workflows/bump_internal_release.yml

@@ -118,10 +118,10 @@ jobs:
           curl -fLSs "https://app.asana.com/api/1.0/tasks/${TASK_ID}?opt_fields=notes" \
             -H "Authorization: Bearer ${ASANA_ACCESS_TOKEN}" \
             | jq -r .data.notes \
-            | ./scripts/extract_release_notes.sh > release_notes.txt
-          release_notes="$(<release_notes.txt)"
-          if [[ ${#release_notes} == 0 || "$release_notes" == "<-- Add release notes here -->" ]]; then
-            echo "::error::Release notes are empty. Please add release notes to the Asana task and restart the workflow."
+            | ./scripts/extract_release_notes.sh -r > raw_release_notes.txt
+          raw_release_notes="$(<raw_release_notes.txt)"
+          if [[ ${#raw_release_notes} == 0 || "$raw_release_notes" == *"<-- Add release notes here -->"* ]]; then
+            echo "::error::Release notes are empty or contain a placeholder. Please add release notes to the Asana task and restart the workflow."
             exit 1
           fi
 

.github/workflows/code_freeze.yml

@@ -172,6 +172,7 @@ jobs:
     uses: ./.github/workflows/tag_release.yml
     with:
       asana-task-url: ${{ needs.create_release_branch.outputs.asana_task_url }}
+      base-branch: ${{ github.ref_name }}
       branch: ${{ needs.create_release_branch.outputs.release_branch_name }}
       prerelease: true
     secrets:

.github/workflows/pr.yml

@@ -54,6 +54,36 @@ jobs:
       env:
         SHELLCHECK_OPTS: -x -P scripts -P scripts/helpers
 
+  bats:
+
+    name: Test Shell Scripts
+
+    runs-on: macos-13
+
+    steps:
+    - name: Check out the code
+      if: github.event_name == 'pull_request' || github.event_name == 'push'
+      uses: actions/checkout@v4
+
+    - name: Check out the code
+      if: github.event_name != 'pull_request' && github.event_name != 'push'
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ inputs.branch || github.ref_name }}
+
+    - name: Install Bats
+      run: brew install bats-core
+
+    - name: Run Bats tests
+      run: bats --formatter junit scripts/tests/* > bats-tests.xml
+
+    - name: Publish unit tests report
+      uses: mikepenz/action-junit-report@v3
+      if: always() # always run even if the previous step fails
+      with:
+        check_name: "Test Report: Shell Scripts"
+        report_paths: 'bats-tests.xml'
+
   tests:
     name: Test
 
@@ -343,7 +373,7 @@ jobs:
 
   create-asana-task:
     name: Create Asana Task
-    needs: [swiftlint, tests, release-build, verify-autoconsent-bundle, private-api]
+    needs: [swiftlint, bats, tests, release-build, verify-autoconsent-bundle, private-api]
 
     if: failure() && github.ref_name == 'main' && github.run_attempt == 1
 
@@ -360,7 +390,7 @@ jobs:
 
   close-asana-task:
     name: Close Asana Task
-    needs: [swiftlint, tests, release-build, verify-autoconsent-bundle, private-api]
+    needs: [swiftlint, bats, tests, release-build, verify-autoconsent-bundle, private-api]
 
     if: success() && github.ref_name == 'main' && github.run_attempt > 1
 

.github/workflows/publish_dmg_release.yml

@@ -145,14 +145,14 @@ jobs:
         run: |
           curl -fLSs "https://app.asana.com/api/1.0/tasks/${TASK_ID}?opt_fields=notes" \
             -H "Authorization: Bearer ${ASANA_ACCESS_TOKEN}" \
-            | jq -r .data.notes \
-            | ./scripts/extract_release_notes.sh > release_notes.txt
-          release_notes="$(<release_notes.txt)"
-          if [[ ${#release_notes} == 0 || "$release_notes" == "<-- Add release notes here -->" ]]; then
-            echo "::error::Release notes are empty. Please add release notes to the Asana task and restart the workflow."
+            | jq -r .data.notes > release_task_content.txt
+          raw_release_notes="$(./scripts/extract_release_notes.sh -r < release_task_content.txt)"
+          if [[ ${#raw_release_notes} == 0 || "$raw_release_notes" == *"<-- Add release notes here -->"* ]]; then
+            echo "::error::Release notes are empty or contain a placeholder. Please add release notes to the Asana task and restart the workflow."
             exit 1
           fi
-          echo "RELEASE_NOTES_FILE=release_notes.txt" >> $GITHUB_ENV
+          ./scripts/extract_release_notes.sh < release_task_content.txt > release_notes.html
+          echo "RELEASE_NOTES_FILE=release_notes.html" >> $GITHUB_ENV
 
       - name: Set up Sparkle tools
         env:
@@ -189,21 +189,21 @@ jobs:
               ./scripts/appcast_manager/appcastManager.swift \
                 --release-to-internal-channel \
                 --dmg ${DMG_PATH} \
-                --release-notes release_notes.txt \
+                --release-notes-html release_notes.html \
                 --key sparkle_private_key
               ;;
             "public")
               ./scripts/appcast_manager/appcastManager.swift \
                 --release-to-public-channel \
                 --version ${VERSION} \
-                --release-notes release_notes.txt \
+                --release-notes-html release_notes.html \
                 --key sparkle_private_key
               ;;
             "hotfix")
               ./scripts/appcast_manager/appcastManager.swift \
                 --release-hotfix-to-public-channel \
                 --dmg ${DMG_PATH} \
-                --release-notes release_notes.txt \
+                --release-notes-html release_notes.html \
                 --key sparkle_private_key
               ;;
             *)

scripts/appcast_manager/appcastManager.swift

@@ -80,6 +80,9 @@ SYNOPSIS
     appcastManager --release-to-internal-channel --dmg <path_to_dmg_file> --release-notes <path_to_release_notes> [--key <path_to_private_key>]
     appcastManager --release-to-public-channel --version <version_identifier> [--release-notes <path_to_release_notes>] [--key <path_to_private_key>]
     appcastManager --release-hotfix-to-public-channel --dmg <path_to_dmg_file> --release-notes <path_to_release_notes> [--key <path_to_private_key>]
+    appcastManager --release-to-internal-channel --dmg <path_to_dmg_file> --release-notes-html <path_to_release_notes_html> [--key <path_to_private_key>]
+    appcastManager --release-to-public-channel --version <version_identifier> [--release-notes-html <path_to_release_notes_html>] [--key <path_to_private_key>]
+    appcastManager --release-hotfix-to-public-channel --dmg <path_to_dmg_file> --release-notes-html <path_to_release_notes_html> [--key <path_to_private_key>]
     appcastManager --help
 
 DESCRIPTION
@@ -109,15 +112,25 @@ DESCRIPTION
     exit(0)
 
 case .releaseToInternalChannel, .releaseHotfixToPublicChannel:
-    guard let dmgPath = arguments.parameters["--dmg"], let releaseNotesPath = arguments.parameters["--release-notes"] else {
+    guard let dmgPath = arguments.parameters["--dmg"] else {
+        print("Missing required parameters")
+        exit(1)
+    }
+    let releaseNotesPath = arguments.parameters["--release-notes"]
+    let releaseNotesHTMLPath = arguments.parameters["--release-notes-html"]
+    guard releaseNotesPath != nil || releaseNotesHTMLPath != nil else {
         print("Missing required parameters")
         exit(1)
     }
     let keyFile = readKeyFileArgument()
 
     print("➡️  Action: Add to internal channel")
     print("➡️  DMG Path: \(dmgPath)")
-    print("➡️  Release Notes Path: \(releaseNotesPath)")
+    if let releaseNotesPath {
+        print("➡️  Release Notes Path: \(releaseNotesPath)")
+    } else if let releaseNotesHTMLPath {
+        print("➡️  Release Notes HTML Path: \(releaseNotesHTMLPath)")
+    }
     if isCI, let keyFile {
         print("➡️  Key file: \(keyFile)")
     }
@@ -130,7 +143,11 @@ case .releaseToInternalChannel, .releaseHotfixToPublicChannel:
     }
 
     // Handle release notes file
-    handleReleaseNotesFile(path: releaseNotesPath, updatesDirectoryURL: specificDir, dmgURL: dmgURL)
+    if let releaseNotesPath {
+        handleReleaseNotesFile(path: releaseNotesPath, updatesDirectoryURL: specificDir, dmgURL: dmgURL)
+    } else if let releaseNotesHTMLPath {
+        handleReleaseNotesHTML(path: releaseNotesHTMLPath, updatesDirectoryURL: specificDir, dmgURL: dmgURL)
+    }
 
     // Extract version number from DMG file name
     let versionNumber = getVersionNumberFromDMGFileName(dmgURL: dmgURL)
@@ -170,6 +187,10 @@ case .releaseToPublicChannel:
         print("Release Notes Path: \(releaseNotesPath)")
         let dmgURLForPublic = specificDir.appendingPathComponent(dmgFileName)
         handleReleaseNotesFile(path: releaseNotesPath, updatesDirectoryURL: specificDir, dmgURL: dmgURLForPublic)
+    } else if let releaseNotesHTMLPath = arguments.parameters["--release-notes-html"] {
+        print("Release Notes Path: \(releaseNotesHTMLPath)")
+        let dmgURLForPublic = specificDir.appendingPathComponent(dmgFileName)
+        handleReleaseNotesHTML(path: releaseNotesHTMLPath, updatesDirectoryURL: specificDir, dmgURL: dmgURLForPublic)
     } else {
         print("👀 No new release notes provided. Keeping existing release notes.")
     }
@@ -605,6 +626,27 @@ final class AppcastDownloader {
 
 // MARK: - Handling of Release Notes
 
+func handleReleaseNotesHTML(path: String, updatesDirectoryURL: URL, dmgURL: URL) {
+    // Copy release notes file and rename it to match the dmg filename
+    let releaseNotesURL = URL(fileURLWithPath: path)
+    let destinationReleaseNotesURL = updatesDirectoryURL.appendingPathComponent(dmgURL.deletingPathExtension().lastPathComponent + ".html")
+
+    do {
+        if FileManager.default.fileExists(atPath: destinationReleaseNotesURL.path) {
+            try FileManager.default.removeItem(at: destinationReleaseNotesURL)
+            print("Old release notes file removed.")
+        }
+
+        // Save the converted release notes to the destination file
+        try FileManager.default.copyItem(at: releaseNotesURL, to: destinationReleaseNotesURL)
+        print("✅ New release notes HTML file copied to the updates directory.")
+
+    } catch {
+        print("❌ Failed to copy and convert release notes HTML file: \(error).")
+        exit(1)
+    }
+}
+
 func handleReleaseNotesFile(path: String, updatesDirectoryURL: URL, dmgURL: URL) {
     // Copy release notes file and rename it to match the dmg filename
     let releaseNotesURL = URL(fileURLWithPath: path)

scripts/extract_release_notes.sh

@@ -7,30 +7,133 @@
 #
 
 start_marker="release notes"
+pp_marker="^for privacy pro subscribers:?$"
 end_marker="this release includes:"
+placeholder="add release notes here"
 is_capturing=0
+is_capturing_pp=0
 has_content=0
+notes=
+pp_notes=
 
-if [[ "$1" == "-t" ]]; then
-    # capture included tasks instead of release notes
-    start_marker="this release includes:"
-    end_marker=
-fi
+output="html"
+
+case "$1" in
+	-a)
+		# Generate Asana rich text output
+		output="asana"
+		;;
+	-r)
+		# Generate raw output instead of HTML
+		output="raw"
+		;;
+	-t)
+		# Capture raw included tasks' URLs instead of release notes
+		output="tasks"
+		start_marker="this release includes:"
+		pp_marker=
+		end_marker=
+		;;
+	*)
+		;;
+esac
+
+html_escape() {
+	local input="$1"
+	sed -e 's/&/\&amp;/g' -e 's/</\&lt;/g' -e 's/>/\&gt;/g' <<< "$input"
+}
+
+make_links() {
+	local input="$1"
+	sed -E 's|(https://[^ ]*)|<a href="\1">\1</a>|' <<< "$input"
+}
+
+lowercase() {
+	local input="$1"
+	tr '[:upper:]' '[:lower:]' <<< "$input"
+}
+
+print_and_exit() {
+	echo -ne "$notes"
+	exit 0
+}
+
+add_to_notes() {
+	notes+="$1"
+	if [[ "$output" != "asana" ]]; then
+		notes+="\\n"
+	fi
+}
+
+add_to_pp_notes() {
+	pp_notes+="$1"
+	if [[ "$output" != "asana" ]]; then
+		pp_notes+="\\n"
+	fi
+}
+
+add_release_note() {
+	local release_note="$1"
+	local processed_release_note=
+	if [[ "$output" == "raw" || "$output" == "tasks" ]]; then
+		processed_release_note="$release_note"
+	else
+		processed_release_note="<li>$(make_links "$(html_escape "$release_note")")</li>"
+	fi
+	if [[ $is_capturing_pp -eq 1 ]]; then
+		add_to_pp_notes "$processed_release_note"
+	else
+		add_to_notes "$processed_release_note"
+	fi
+}
 
 while read -r line
 do
-    if [[ $(tr '[:upper:]' '[:lower:]' <<< "$line") == "$start_marker" ]]; then
-        is_capturing=1
-    elif [[ -n "$end_marker" && $(tr '[:upper:]' '[:lower:]' <<< "$line") == "$end_marker" ]]; then
-        exit 0
-    elif [[ $is_capturing -eq 1 && -n "$line" ]]; then
-        has_content=1
-        echo "$line"
-    fi
+	# Lowercase each line to compare with markers
+	lowercase_line="$(lowercase "$line")"
+
+	if [[ "$lowercase_line" == "$start_marker" ]]; then
+		# Only start capturing here
+		is_capturing=1
+		if [[ "$output" == "asana" ]]; then
+			add_to_notes "<ul>"
+		elif [[ "$output" == "html" ]]; then
+			# Add HTML header and start the list
+			add_to_notes "<h3 style=\"font-size:14px\">What's new</h3>"
+			add_to_notes "<ul>"
+		fi
+	elif [[ -n "$pp_marker" && "$lowercase_line" =~ $pp_marker ]]; then
+		is_capturing_pp=1
+		if [[ "$output" == "asana" ]]; then
+			add_to_pp_notes "</ul><h2>For Privacy Pro subscribers</h2><ul>"
+		elif [[ "$output" == "html" ]]; then
+			# If we've reached the PP marker, end the list and start the PP list
+			add_to_pp_notes "</ul>"
+			add_to_pp_notes "<h3 style=\"font-size:14px\">For Privacy Pro subscribers</h3>"
+			add_to_pp_notes "<ul>"
+		else
+			add_to_pp_notes "$line"
+		fi
+	elif [[ -n "$end_marker" && "$lowercase_line" == "$end_marker" ]]; then
+		# If we've reached the end marker, check if PP notes are present and not a placeholder, and add them verbatim to notes
+		# shellcheck disable=SC2076
+		if [[ -n "$pp_notes" && ! "$(lowercase "$pp_notes")" =~ "$placeholder" ]]; then
+			notes+="$pp_notes" # never add extra newline here (that's why we don't use `add_to_notes`)
+		fi
+		if [[ "$output" != "raw" ]]; then
+			# End the list on end marker
+			add_to_notes "</ul>"
+		fi
+		# Print output and exit
+		print_and_exit
+	elif [[ $is_capturing -eq 1 && -n "$line" ]]; then
+		has_content=1
+		add_release_note "$line"
+	fi
 done
 
 if [[ $has_content -eq 0 ]]; then
-    exit 1
+	exit 1
 fi
 
-exit 0
+print_and_exit