Update pre-commit (#919)

* Update pre-commit
- separate pre-commit and pre-push
- fix lint issues in grafana

* fix some shell check issues

* review comments and simplify lint
also add tidy check

* Add installs to make and nix for installing the pre-push hook

* add more default checks

* Fix some whitespace issues and remove yaml

* Fix whitespace issues in charts

* Add pre-commit checks to ci

* Rename to be more clear and consistent

* Fix mock server

* test

* add unit tests back in

* use version itself

.githooks/detect-ethereum-keys

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Define the regex pattern for Ethereum keys
+ethereum_key_pattern="(0x[a-fA-F0-9]{40})"
+
+# Loop over each file passed to the script
+for file in "$@"; do
+    echo -n "Checking $file for ethereum keys... "
+
+    if grep -q "$ethereum_key_pattern" "$file"; then
+        echo -e "❌\nFound eth keys in $file:"
+        grep -n "$ethereum_key_pattern" "$file"
+        echo -e "\e[31mRemove the Ethereum key before committing.\e[0m\n"
+        exit 1
+    else
+        echo "✅"
+    fi
+done

.githooks/detect-rpc-urls

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Define the regex pattern for HTTP and WS URLs
+url_pattern="(http|ws)(s)?:\/\/[^ \t\n\r]+"
+
+# Loop over each file passed to the script
+for file in "$@"; do
+    echo -n "Checking $file for RPC URLs... "
+    if grep -q "$url_pattern" "$file"; then
+        echo -e "❌\nFound RPC URL in $file:"
+        grep -n "$url_pattern" "$file"
+        echo -e "\e[31mRemove the RPC URL before committing.\e[0m\n"
+        exit 1
+    else
+        echo "✅"
+    fi
+done

.githooks/go-lint

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Find all 'go.mod' files, get their directories, and run 'go mod tidy'
+# shellcheck disable=SC2016
+find "./" -type f -name 'go.mod' -print0 | xargs -0 -I{} bash -c '
+    directory=$(dirname "{}")
+    cd "$directory" || exit 1
+
+    # Run linter and capture exit status
+    golangci-lint run  > /dev/null
+    local linting_result=$?
+
+    # Check linting result
+    if [[ $linting_result -ne 0 ]]; then
+        echo -e "Executing linters in $directory... \e[31mNOK!\e[0m\n"
+        echo -e "Run \`cd $directory && golangci-lint run -fix -v\` and fix the issues\n"
+        exit 1
+    else
+        echo -e "Executing linters in $directory... \e[32mOK!\e[0m\n"
+    fi
+'

.githooks/go-mod-local-replace

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# Loop over each file passed to the script
+for file in "$@"; do
+    echo -n "Checking $file for local replacements... "
+    if grep -q 'replace .* => /' "$file"; then
+        echo -e "❌\nFound local replacements in $file:"
+        grep -n 'replace .* => /' "$file"
+        echo -e "\e[31mYou forgot about a local replacement\e[0m\n"
+        exit 1
+    else
+        echo "✅"
+    fi
+done

.githooks/go-mod-tidy

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+# Find all 'go.mod' files, get their directories, and run 'go mod tidy'
+# shellcheck disable=SC2016
+find "./" -type f -name 'go.mod' -print0 | xargs -0 -I{} bash -c '
+    dir=$(dirname "{}")
+    echo "Executing cd \"$dir\" && go mod tidy"
+    cd "$dir"
+    go mod tidy
+'
+# pre-commit stashes changes before running the hooks so we can use git to check for changes here
+# Run git diff and capture output
+output=$(git diff --stat)
+
+if [ -z "$output" ]; then
+    echo "No changes in any files."
+else
+    echo "go.mod files that need to be tidied:"
+    echo "$output" | awk -F '|' '/\|/ { print $1 }'
+    exit 1
+fi

.githooks/go-test-build

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+# Find all 'go.mod' files, get their directories, and run an empty 'go test' in them to compile the tests.
+# shellcheck disable=SC2016
+find "./" -type f -name 'go.mod' -print0 | xargs -0 -I{} bash -c '
+    dir=$(dirname "{}")
+    echo "Executing cd \"$dir\" && go test -run=^# ./..."
+    cd "$dir"
+    go test -run=^# ./...
+'

.githooks/run-unit-tests

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+printf "Executing unit tests..."
+
+make test_unit > /dev/null 2>&1
+unit_test_result=$?
+
+if [[ $unit_test_result -ne 0 ]];
+then
+    printf "Executing unit tests... ❌\n"
+    printf "Run 'make test_unit' to verify after you fix them.\n"
+    exit 1
+else
+    printf "Executing unit tests... ✅\n"
+fi

.github/workflows/lint.yaml

@@ -6,6 +6,18 @@ on:
   pull_request:
 
 jobs:
+  pre-commit:
+    name: Pre-commit checks
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the Repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Install Nix
+        uses: cachix/install-nix-action@8887e596b4ee1134dae06b98d573bd674693f47c # v26
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+      - name: Run pre-commit checks
+        run: nix develop -c sh -c "pre-commit run --hook-stage pre-commit --show-diff-on-failure --color=always --all-files"
   tools:
     name: Get tool-versions
     runs-on: ubuntu-latest
@@ -86,18 +98,6 @@ jobs:
       - name: Install asdf dependencies
         uses: asdf-vm/actions/install@05e0d2ed97b598bfce82fd30daf324ae0c4570e6 # v3.0.2
 
-  prettier:
-    name: Formatting
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout the repo
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-      - name: Run Prettier
-        uses: creyD/prettier_action@31355f8eef017f8aeba2e0bc09d8502b13dbbad1 # v4.3
-        with:
-          dry: true
-          prettier_options: --write **/*.{yaml,yml,json} --config ./.prettierrc.yaml --ignore-path ./.prettierignore
-
   helmlint:
     name: Lint Helm Charts
     runs-on: ubuntu-latest

.gitignore

@@ -69,7 +69,7 @@ e2e.test
 k3dvolume/
 charts/**/*.tgz
 __debug*
-# saved private chain configuration 
+# saved private chain configuration
 .private_chains/*
 
 .direnv

.nancy-ignore

@@ -10,4 +10,4 @@ CVE-2023-3518 # CWE-Other (when using JWT Auth for service mesh incorrectly allo
 CVE-2023-40591 # CWE-400 Uncontrolled Resource Consumption ('Resource Exhaustion') by geth (fixed in v1.12.2, which needs core to bump that dep first)
 CVE-2023-42319 # CWE-400 Uncontrolled Resource Consumption ('Resource Exhaustion') by geth (fixed in v1.12.2, which needs core to bump that dep first)
 CVE-2023-48795 # Improper Validation of Integrity Check Value x/crypto, which is an indirect dependency
-CVE-2024-24786 # CWE-835 Loop with Unreachable Exit Condition ('Infinite Loop') in indrect dependency
+CVE-2024-24786 # CWE-835 Loop with Unreachable Exit Condition ('Infinite Loop') in indrect dependency

.pre-commit-config.yaml

@@ -1,4 +1,42 @@
 repos:
+  - repo: local
+    hooks:
+      - id: go-lint
+        name: Check go lint
+        entry: ./.githooks/go-lint
+        language: script
+        pass_filenames: false
+      - id: go-mod-local-replace
+        name: Check for local replaces in go.mod
+        entry: ./.githooks/go-mod-local-replace
+        language: script
+        files: 'go\.mod$'
+      - id: detect-etherum-keys
+        name: Detect ethereum keys
+        entry: ./.githooks/detect-ethereum-keys
+        language: script
+      - id: detect-rpc-urls
+        name: Check for RPC URLs
+        entry: ./.githooks/detect-rpc-urls
+        language: script
+      - id: go-mod-tidy
+        name: Check go mod tidy
+        entry: ./.githooks/go-mod-tidy
+        language: script
+        pass_filenames: false
+        stages: [pre-push]
+      - id: go-test-build
+        name: Check go test build
+        entry: ./.githooks/go-test-build
+        language: script
+        pass_filenames: false
+        stages: [pre-push]
+      - id: run-unit-tests
+        name: Run unit tests
+        entry: ./.githooks/run-unit-tests
+        language: script
+        pass_filenames: false
+        stages: [pre-push]
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: 2c9f875913ee60ca25ce70243dc24d5b6415598c # v4.6.0
     hooks:
@@ -7,22 +45,22 @@ repos:
         args: [--allow-missing-credentials]
       - id: check-added-large-files
       - id: check-json
+      - id: check-toml
+      - id: end-of-file-fixer
+      - id: check-case-conflict
+      - id: check-shebang-scripts-are-executable
+      - id: check-merge-conflict
+      - id: trailing-whitespace
   - repo: https://github.com/gruntwork-io/pre-commit
     rev: e9250bd69bb312d55364213ff5ff037a09be55d9 # v0.1.12
     hooks:
-      - name: Check Helm charts
-        id: helmlint
+      # TODO: change to using the same lint a in CI
+      # - name: Check Helm charts
+      #   id: helmlint
       - name: Check shell scripts
         id: shellcheck
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: bc48c541add1551be726f23c4294c773442341cb # v2.3.2
     hooks:
       - name: Prettier
         id: prettier
-  - repo: local
-    hooks:
-      - id: golangci-lint
-        name: golangci-lint
-        entry: golangci-lint run
-        language: system
-        types: [go]

.tool-versions

@@ -2,12 +2,12 @@ golang 1.21.5
 nodejs 14.20.0
 k3d 5.5.1
 act 0.2.52
-golangci-lint 1.57.1
+golangci-lint 1.57.2
 actionlint 1.6.26
 shellcheck 0.9.0
 helm 3.13.1
 kubectl 1.25.5
 yarn 1.22.19
 python 3.9.13
 pre-commit 3.5.0
-gcloud 466.0.0
+gcloud 466.0.0

CODEOWNERS

@@ -1 +1 @@
-* @sebawo @smartcontractkit/test-tooling-team
+* @sebawo @smartcontractkit/test-tooling-team

Dockerfile.compiler

@@ -16,4 +16,4 @@ COPY ${projectRootPath}/. .
 RUN go test -c -ldflags="-s -w -extldflags=-static" -o ./remote.test ${testDirectory}
 
 FROM scratch AS export
-COPY --from=compiler /app/remote.test .
+COPY --from=compiler /app/remote.test .

Makefile

@@ -19,15 +19,11 @@ else
 endif
 
 lint:
-	golangci-lint --color=always run ./... --fix -v
-	cd ./tools/gotestloghelper && golangci-lint --color=always run ./... --fix -v && cd -
-	cd ./k8s-test-runner && golangci-lint --color=always run ./... --fix -v && cd -
+	pre-commit run go-lint --all-files --show-diff-on-failure --color=always
 
 .PHONY: tidy
 tidy:
-	go mod tidy
-	cd ./tools/gotestloghelper && go mod tidy && cd -
-	cd ./k8s-test-runner && go mod tidy && cd -
+	pre-commit run go-mod-tidy --all-files --show-diff-on-failure --color=always
 
 .PHONY: go_mod
 go_mod:
@@ -62,6 +58,7 @@ ifeq ($(OSFLAG),$(OSX))
 	helm repo add grafana https://grafana.github.io/helm-charts
 	helm repo update
 	pre-commit install
+	pre-commit install --hook-type pre-push
 endif
 
 install: go_mod install_tools
@@ -72,9 +69,6 @@ docker_prune:
 	docker system prune -a -f
 	docker volume prune -f
 
-compile_contracts:
-	python3 ./utils/compile_contracts.py
-
 test_unit: go_mod
 	go test -timeout 5m -json -cover -covermode=count -coverprofile=unit-test-coverage.out $(shell go list ./... | grep -v /k8s/e2e/ | grep -v /k8s/examples/ | grep -v /docker/test_env) 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci
 

charts/besu-prysm/.helmignore

@@ -1,2 +1,2 @@
 *.sh
-besu-prysm-*.tgz
+besu-prysm-*.tgz

charts/besu-prysm/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 description: Eth2 private network with Besu and Prysm
 name: besu-prysm
-version: 0.2.2
+version: 0.2.3
 dependencies:
   - name: eth2-common
     version: 0.5.2

charts/besu-prysm/README.md

@@ -96,4 +96,4 @@ It's recommended to remove the installation with `./uninstall.sh` script, as it
 # Limitations
 * No support for restarting of beacon chain pod (validator gets slashed)
 * Untested scalability
-* no working readiness/liveliness probe for beacon chain
+* no working readiness/liveliness probe for beacon chain