From c551820f848f5dfef382536d792b44a596cd9d23 Mon Sep 17 00:00:00 2001 From: Peter Sanderson Date: Wed, 7 Aug 2024 10:55:32 +0200 Subject: [PATCH] Add check to keep releases.json in sync with the firmware binaries --- .github/workflows/check-shell-validation.yml | 19 ++++++ .github/workflows/check_releases.yml | 9 +++ ci/s3sync.sh | 10 ++- firmware/t3b1/releases.json | 1 + ...heck-firmware-presence-in-releases-json.sh | 62 +++++++++++++++++++ scripts/run-releases-json-for-all-devices.sh | 13 ++++ scripts/shellcheck.sh | 10 +++ 7 files changed, 118 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/check-shell-validation.yml create mode 100644 firmware/t3b1/releases.json create mode 100755 scripts/check-firmware-presence-in-releases-json.sh create mode 100755 scripts/run-releases-json-for-all-devices.sh create mode 100755 scripts/shellcheck.sh diff --git a/.github/workflows/check-shell-validation.yml b/.github/workflows/check-shell-validation.yml new file mode 100644 index 0000000..ea647da --- /dev/null +++ b/.github/workflows/check-shell-validation.yml @@ -0,0 +1,19 @@ +name: "[Check]: Shell validation" + +on: + pull_request: + paths: + - "**.sh" + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + shellcheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: shellcheck + run: ./scripts/shellcheck.sh diff --git a/.github/workflows/check_releases.yml b/.github/workflows/check_releases.yml index c14561c..6bf5214 100644 --- a/.github/workflows/check_releases.yml +++ b/.github/workflows/check_releases.yml @@ -12,6 +12,15 @@ jobs: - uses: actions/setup-python@v4 - run: python check_releases.py + releases-json-integrity-check: + name: releases.json integrity check + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Check releases.json files changes + run: ./scripts/run-releases-json-for-all-devices.sh + releases-revision-checks: name: Releases revision Checks runs-on: ubuntu-latest diff --git a/ci/s3sync.sh b/ci/s3sync.sh index ced114d..60d05c2 100755 --- a/ci/s3sync.sh +++ b/ci/s3sync.sh @@ -6,11 +6,9 @@ DIRS="bootloader bridge firmware legal registry udev suite connect security transparency misc" BUCKET=data.trezor.io -ROLLBACK=rollback-data.trezor.io DISTRIBUTION_ID="E1ERY5K2OTKKI1" -./check_releases.py -if [ "$?" != "0" ]; then +if ! ./check_releases.py; then echo "check_releases.py failed." exit fi @@ -21,10 +19,10 @@ set -e # aws s3 sync s3://$BUCKET s3://$ROLLBACK for DIR in $DIRS; do - if [ "x$1" == "x-d" ]; then - aws s3 sync --delete --cache-control 'public, max-age=3600' $DIR s3://$BUCKET/$DIR + if [ "$1" == "-d" ]; then + aws s3 sync --delete --cache-control 'public, max-age=3600' "$DIR" s3://$BUCKET/"$DIR" else - aws s3 sync --cache-control 'public, max-age=3600' $DIR s3://$BUCKET/$DIR + aws s3 sync --cache-control 'public, max-age=3600' "$DIR" s3://$BUCKET/"$DIR" fi done diff --git a/firmware/t3b1/releases.json b/firmware/t3b1/releases.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/firmware/t3b1/releases.json @@ -0,0 +1 @@ +[] diff --git a/scripts/check-firmware-presence-in-releases-json.sh b/scripts/check-firmware-presence-in-releases-json.sh new file mode 100755 index 0000000..0ee2eac --- /dev/null +++ b/scripts/check-firmware-presence-in-releases-json.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +PARENT_PATH=$( cd "$(dirname "${BASH_SOURCE[0]}")" || exit ; pwd -P ) + +GREEN='\033[0;32m' +RED='\033[0;31m' +NC='\033[0m' # No Color + +if [[ $# -ne 1 ]] + then + echo "must provide 1 argument. $# provided" + exit 1 +fi + +DEVICE=$1 + +extract_filenames_from_json() { + local json_file="$1" + jq -r '.[] | select(.url) | .url, .url_bitcoinonly' "$json_file" | xargs -n 1 basename | sort | uniq \ + | grep -v "null" # filter out null from missing .url_bitcoinonly for older firmwares +} + +list_files_in_directory() { + local dir="$1" + find "$dir" -type f -name "*.bin" -exec basename {} \; | sort \ + | grep -v "trezor-inter-" | grep -v "trezor-t1tb-inter-" # filer out Intermediary firmwares +} + +compare_files() { + local json_file="$1" + local directory="$2" + + expected_files=$(extract_filenames_from_json "$json_file") + actual_files=$(list_files_in_directory "$directory") + + missing_files=$(comm -23 <(echo "$expected_files") <(echo "$actual_files")) + extra_files=$(comm -13 <(echo "$expected_files") <(echo "$actual_files")) + + if [[ -z "$missing_files" && -z "$extra_files" ]]; then + echo -e "${GREEN}All files are present and accounted for.${NC}" + else + if [[ -n "$missing_files" ]]; then + echo -e "${RED}Missing files:" + echo "$missing_files" | awk '{print " " $0}' + echo -e "${NC}" + fi + if [[ -n "$extra_files" ]]; then + echo -e "${RED}Extra files in directory:" + echo "$extra_files" | awk '{print " " $0}' + echo -e "${NC}" + fi + + exit 1 + fi +} + +json_file=$PARENT_PATH"/../firmware/"$DEVICE/"releases.json" +directory=$PARENT_PATH"/../firmware/"$DEVICE + +echo "Checking directory: $directory" + +compare_files "$json_file" "$directory" diff --git a/scripts/run-releases-json-for-all-devices.sh b/scripts/run-releases-json-for-all-devices.sh new file mode 100755 index 0000000..b3f3ed5 --- /dev/null +++ b/scripts/run-releases-json-for-all-devices.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +DEVICE_PATHS=$(find firmware -maxdepth 1 -type d ! -name 'translations' ! -name 'README.md' ! -name 'firmware') + +for FILE in $DEVICE_PATHS; + do + DEVICE_MODEL=$(basename "$FILE") + if ! ./scripts/check-firmware-presence-in-releases-json.sh "$DEVICE_MODEL" ; then + exit 1 + fi; + + echo + done diff --git a/scripts/shellcheck.sh b/scripts/shellcheck.sh new file mode 100755 index 0000000..9254743 --- /dev/null +++ b/scripts/shellcheck.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -e +set -u +set -x +set -o pipefail + +shellcheck --version + +find . -type f -name '*.sh' -exec shellcheck {} +