diff --git a/.github/workflows/nightlyfuzz.yml b/.github/workflows/nightlyfuzz.yml new file mode 100644 index 00000000000..7becbe73de5 --- /dev/null +++ b/.github/workflows/nightlyfuzz.yml @@ -0,0 +1,53 @@ +name: 'nightly/tag fuzz' +on: + schedule: + # Note: The schedule event can be delayed during periods of high + # loads of GitHub Actions workflow runs. High load times include + # the start of every hour. To decrease the chance of delay, + # schedule your workflow to run at a different time of the hour. + - cron: "25 0 * * *" # at 25 past midnight every day + push: + tags: + - '*' + workflow_dispatch: null +jobs: + fuzzrun: + name: "run native fuzzers" + runs-on: "ubuntu20.04-4cores-16GB" + steps: + - name: "Checkout" + uses: "actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11" # v4.1.1 + - name: "Setup go" + uses: "actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491" # v5.0.0 + with: + go-version-file: 'go.mod' + cache: true + cache-dependency-path: 'go.sum' + - name: "Get corpus directory" + id: "get-corpus-dir" + run: echo "corpus_dir=$(go env GOCACHE)/fuzz" >> $GITHUB_OUTPUT + shell: "bash" + - name: "Restore corpus" + uses: "actions/cache/restore@13aacd865c20de90d75de3b17ebe84f7a17d57d2" # v4.0.0 + id: "restore-corpus" + with: + path: "${{ steps.get-corpus-dir.outputs.corpus_dir }}" + # We need to ensure uniqueness of the key, as saving to a key more than once will fail (see Save corpus step). + # We never expect a cache hit with the key but we do expect a hit with the restore-keys prefix that is going + # to match the latest cache that has that prefix. + key: "nightlyfuzz-corpus-${{ github.run_id }}-${{ github.run_attempt }}" + restore-keys: "nightlyfuzz-corpus-" + - name: "Run native fuzzers" + # Fuzz for 1 hour + run: "cd fuzz && ./fuzz_all_native.py --seconds 3600" + - name: "Print failing testcases" + if: failure() + run: find . -type f|fgrep '/testdata/fuzz/'|while read f; do echo $f; cat $f; done + - name: "Save corpus" + uses: "actions/cache/save@13aacd865c20de90d75de3b17ebe84f7a17d57d2" # v4.0.0 + # We save also on failure, so that we can keep the valuable corpus generated that led to finding a crash. + # If the corpus gets clobbered for any reason, we can remove the offending cache from the Github UI. + if: always() + with: + path: "${{ steps.get-corpus-dir.outputs.corpus_dir }}" + key: "${{ steps.restore-corpus.outputs.cache-primary-key }}" \ No newline at end of file diff --git a/fuzz/.gitignore b/fuzz/.gitignore new file mode 100644 index 00000000000..600ab25dfaf --- /dev/null +++ b/fuzz/.gitignore @@ -0,0 +1 @@ +*/fuzzer \ No newline at end of file diff --git a/fuzz/fuzz_all_native.py b/fuzz/fuzz_all_native.py new file mode 100755 index 00000000000..41588b090b7 --- /dev/null +++ b/fuzz/fuzz_all_native.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +import argparse +import itertools +import os +import re +import subprocess +import sys + +LIBROOT = "../" + +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description="\n".join([ + "Fuzz helper to run all native go fuzzers in chainlink", + "", + ]), + ) + parser.add_argument("--ci", required=False, help="In CI mode we run each parser only briefly once", action="store_true") + parser.add_argument("--seconds", required=False, help="Run for this many seconds of total fuzz time before exiting") + args = parser.parse_args() + + # use float for remaining_seconds so we can represent infinity + if args.seconds: + remaining_seconds = float(args.seconds) + else: + remaining_seconds = float("inf") + + fuzzers = discover_fuzzers() + print(f"🐝 Discovered fuzzers:", file=sys.stderr) + for fuzzfn, path in fuzzers.items(): + print(f"{fuzzfn} in {path}", file=sys.stderr) + + if args.ci: + # only run each fuzzer once for 60 seconds in CI + durations_seconds = [60] + else: + # run forever or until --seconds, with increasingly longer durations per fuzz run + durations_seconds = itertools.chain([5, 10, 30, 90, 270], itertools.repeat(600)) + + for duration_seconds in durations_seconds: + print(f"🐝 Running each fuzzer for {duration_seconds}s before switching to next fuzzer", file=sys.stderr) + for fuzzfn, path in fuzzers.items(): + if remaining_seconds <= 0: + print(f"🐝 Time budget of {args.seconds}s is exhausted. Exiting.", file=sys.stderr) + return + + next_duration_seconds = min(remaining_seconds, duration_seconds) + remaining_seconds -= next_duration_seconds + + print(f"🐝 Running {fuzzfn} in {path} for {next_duration_seconds}s before switching to next fuzzer", file=sys.stderr) + run_fuzzer(fuzzfn, path, next_duration_seconds) + print(f"🐝 Completed running {fuzzfn} in {path} for {next_duration_seconds}s. Total remaining time is {remaining_seconds}s", file=sys.stderr) + +def discover_fuzzers(): + fuzzers = {} + for root, dirs, files in os.walk(LIBROOT): + for file in files: + if not file.endswith("test.go"): continue + with open(os.path.join(root, file), "r") as f: + text = f.read() + # ignore multiline comments + text = re.sub(r"(?s)/[*].*?[*]/", "", text) + # ignore single line comments *except* build tags + text = re.sub(r"//.*", "", text) + # Find every function with a name like FuzzXXX + for fuzzfn in re.findall(r"func\s+(Fuzz\w+)", text): + if fuzzfn in fuzzers: + raise Exception(f"Duplicate fuzz function: {fuzzfn}") + fuzzers[fuzzfn] = os.path.relpath(root, LIBROOT) + return fuzzers + +def run_fuzzer(fuzzfn, dir, duration_seconds): + subprocess.check_call(["go", "test", "-run=^$", f"-fuzz=^{fuzzfn}$", f"-fuzztime={duration_seconds}s", f"./{dir}"], cwd=LIBROOT) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/sonar-project.properties b/sonar-project.properties index db2a6be1f3b..f522b3fa73c 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -3,7 +3,7 @@ sonar.sources=. sonar.python.version=3.8 # Full exclusions from the static analysis -sonar.exclusions=**/node_modules/**/*,**/mocks/**/*, **/testdata/**/*, **/contracts/typechain/**/*, **/contracts/artifacts/**/*, **/contracts/cache/**/*, **/contracts/scripts/**/*, **/generated/**/*, **/fixtures/**/*, **/docs/**/*, **/tools/**/*, **/*.pb.go, **/*report.xml, **/*.config.ts, **/*.txt, **/*.abi, **/*.bin, **/*_codecgen.go, core/services/relay/evm/types/*_gen.go, core/services/relay/evm/types/gen/main.go, core/services/relay/evm/testfiles/*, **/core/web/assets**, core/scripts/chaincli/handler/debug.go +sonar.exclusions=**/node_modules/**/*,**/mocks/**/*, **/testdata/**/*, **/contracts/typechain/**/*, **/contracts/artifacts/**/*, **/contracts/cache/**/*, **/contracts/scripts/**/*, **/generated/**/*, **/fixtures/**/*, **/docs/**/*, **/tools/**/*, **/*.pb.go, **/*report.xml, **/*.config.ts, **/*.txt, **/*.abi, **/*.bin, **/*_codecgen.go, core/services/relay/evm/types/*_gen.go, core/services/relay/evm/types/gen/main.go, core/services/relay/evm/testfiles/*, **/core/web/assets**, core/scripts/chaincli/handler/debug.go, **/fuzz/**/* # Coverage exclusions sonar.coverage.exclusions=**/*.test.ts, **/*_test.go, **/contracts/test/**/*, **/contracts/**/tests/**/*, **/core/**/testutils/**/*, **/core/**/mocks/**/*, **/core/**/cltest/**/*, **/integration-tests/**/*, **/generated/**/*, **/core/scripts**/* , **/*.pb.go, ./plugins/**/*, **/main.go, **/0195_add_not_null_to_evm_chain_id_in_job_specs.go, **/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/mercury/streams/streams.go # Duplication exclusions diff --git a/tools/bin/go_core_fuzz b/tools/bin/go_core_fuzz index 3ea7d9bb0cb..2ee251b5ec5 100755 --- a/tools/bin/go_core_fuzz +++ b/tools/bin/go_core_fuzz @@ -6,9 +6,8 @@ SCRIPT_PATH=`dirname "$0"`; SCRIPT_PATH=`eval "cd \"$SCRIPT_PATH\" && pwd"` OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} USE_TEE="${USE_TEE:-true}" -echo "Failed tests and panics: ---------------------" +echo "Failed fuzz tests and panics: ---------------------" echo "" -GO_LDFLAGS=$(bash tools/bin/ldflags) use_tee() { if [ "$USE_TEE" = "true" ]; then tee "$@" @@ -16,7 +15,10 @@ use_tee() { cat > "$@" fi } -go test -json -ldflags "$GO_LDFLAGS" github.com/smartcontractkit/chainlink/v2/core/services/relay/evm -fuzz . -fuzztime 12s | use_tee $OUTPUT_FILE + +# the amount of --seconds here is subject to change based on how long the CI job takes in the future +# as we add more fuzz tests, we should take into consideration increasing this timelapse, so we can have enough coverage. +cd ./fuzz && ./fuzz_all_native.py --ci --seconds 420 | use_tee $OUTPUT_FILE EXITCODE=${PIPESTATUS[0]} # Assert no known sensitive strings present in test logger output @@ -29,8 +31,9 @@ fi echo "Exit code: $EXITCODE" if [[ $EXITCODE != 0 ]]; then - echo "Encountered test failures." + echo "Encountered fuzz test failures. Logging all failing fuzz inputs:" + find . -type f|fgrep '/testdata/fuzz/'|while read f; do echo $f; cat $f; done else - echo "All tests passed!" + echo "All fuzz tests passed!" fi exit $EXITCODE