Merge branch 'develop' into universal_estimator_polygon_test

.changeset/tall-emus-fail.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+#internal Use txid as the idempotency key in the evm chainwriter

core/services/relay/evm/chain_writer.go

@@ -128,6 +128,7 @@ func (w *chainWriter) SubmitTransaction(ctx context.Context, contract, method st
 		EncodedPayload: calldata,
 		FeeLimit:       methodConfig.GasLimit,
 		Meta:           txMeta,
+		IdempotencyKey: &transactionID,
 		Strategy:       w.sendStrategy,
 		Checker:        checker,
 		Value:          *v,

fuzz/fuzz_all_native.py

@@ -7,8 +7,6 @@
 import subprocess
 import sys
 
-LIBROOT = "../"
-
 def main():
     parser = argparse.ArgumentParser(
         formatter_class=argparse.RawDescriptionHelpFormatter,
@@ -19,6 +17,7 @@ def main():
     )
     parser.add_argument("--ci", required=False, help="In CI mode we run each parser only briefly once", action="store_true")
     parser.add_argument("--seconds", required=False, help="Run for this many seconds of total fuzz time before exiting")
+    parser.add_argument("--go_module_root", required=True, help="Path to the root of the go module to fuzz")
     args = parser.parse_args()
 
     # use float for remaining_seconds so we can represent infinity
@@ -27,7 +26,7 @@ def main():
     else:
         remaining_seconds = float("inf")
 
-    fuzzers = discover_fuzzers()
+    fuzzers = discover_fuzzers(args.go_module_root)
     print(f"🐝 Discovered fuzzers:", file=sys.stderr)
     for fuzzfn, path in fuzzers.items():
         print(f"{fuzzfn} in {path}", file=sys.stderr)
@@ -50,12 +49,12 @@ def main():
             remaining_seconds -= next_duration_seconds
 
             print(f"🐝 Running {fuzzfn} in {path} for {next_duration_seconds}s before switching to next fuzzer", file=sys.stderr)
-            run_fuzzer(fuzzfn, path, next_duration_seconds)
+            run_fuzzer(fuzzfn, path, next_duration_seconds, args.go_module_root)
             print(f"🐝 Completed running {fuzzfn} in {path} for {next_duration_seconds}s. Total remaining time is {remaining_seconds}s", file=sys.stderr)
 
-def discover_fuzzers():
+def discover_fuzzers(go_module_root):
     fuzzers = {}
-    for root, dirs, files in os.walk(LIBROOT):
+    for root, dirs, files in os.walk(go_module_root):
         for file in files:
             if not file.endswith("test.go"): continue
             with open(os.path.join(root, file), "r") as f:
@@ -68,11 +67,11 @@ def discover_fuzzers():
             for fuzzfn in re.findall(r"func\s+(Fuzz\w+)", text):
                 if fuzzfn in fuzzers:
                     raise Exception(f"Duplicate fuzz function: {fuzzfn}")
-                fuzzers[fuzzfn] = os.path.relpath(root, LIBROOT)
+                fuzzers[fuzzfn] = os.path.relpath(root, go_module_root)
     return fuzzers
 
-def run_fuzzer(fuzzfn, dir, duration_seconds):
-    subprocess.check_call(["go", "test", "-run=^$", f"-fuzz=^{fuzzfn}$", f"-fuzztime={duration_seconds}s", f"./{dir}"], cwd=LIBROOT)
+def run_fuzzer(fuzzfn, dir, duration_seconds, go_module_root):
+    subprocess.check_call(["go", "test", "-run=^$", f"-fuzz=^{fuzzfn}$", f"-fuzztime={duration_seconds}s", f"./{dir}"], cwd=go_module_root)
 
 if __name__ == "__main__":
     main()

integration-tests/go.mod

@@ -29,7 +29,7 @@ require (
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-automation v1.0.4
 	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240710165532-ade916a95858
-	github.com/smartcontractkit/chainlink-testing-framework v1.32.0
+	github.com/smartcontractkit/chainlink-testing-framework v1.32.2
 	github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239
 	github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000
 	github.com/smartcontractkit/havoc/k8schaos v0.0.0-20240409145249-e78d20847e37

integration-tests/go.sum

@@ -1526,8 +1526,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240710170818-eccca28888e
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240710170818-eccca28888e5/go.mod h1:aJUY4hdo1g942mhlPX9Z4FWe5ldEyWvsWSNf7frh7yU=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799 h1:HyLTySm7BR+oNfZqDTkVJ25wnmcTtxBBD31UkFL+kEM=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799/go.mod h1:UVFRacRkP7O7TQAzFmR52v5mUlxf+G1ovMlCQAB/cHU=
-github.com/smartcontractkit/chainlink-testing-framework v1.32.0 h1:6emeE/YuMNmdd1pZ3NlJGu94QCa+NUayx26NX0jAQDY=
-github.com/smartcontractkit/chainlink-testing-framework v1.32.0/go.mod h1:wSRZGoukZliwfTkghdF4cI1RLHz5k3aDnL+rXhJ6f7k=
+github.com/smartcontractkit/chainlink-testing-framework v1.32.2 h1:j5OQ9Xt2aqAGJfpEkh1pxvOiZMVbxypjFEBZkdIEfZg=
+github.com/smartcontractkit/chainlink-testing-framework v1.32.2/go.mod h1:OYlILtWsQskqZgzIjRe7PpIEySJVhb4qMnGxOuzrUgA=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 h1:Kk5OVlx/5g9q3Z3lhxytZS4/f8ds1MiNM8yaHgK3Oe8=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU=
 github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo=

integration-tests/load/go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-automation v1.0.4
 	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240710165532-ade916a95858
-	github.com/smartcontractkit/chainlink-testing-framework v1.32.0
+	github.com/smartcontractkit/chainlink-testing-framework v1.32.2
 	github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c
 	github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8
 	github.com/smartcontractkit/libocr v0.0.0-20240702141926-063ceef8c42e

integration-tests/load/go.sum

@@ -1516,8 +1516,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240710170818-eccca28888e
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240710170818-eccca28888e5/go.mod h1:aJUY4hdo1g942mhlPX9Z4FWe5ldEyWvsWSNf7frh7yU=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799 h1:HyLTySm7BR+oNfZqDTkVJ25wnmcTtxBBD31UkFL+kEM=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799/go.mod h1:UVFRacRkP7O7TQAzFmR52v5mUlxf+G1ovMlCQAB/cHU=
-github.com/smartcontractkit/chainlink-testing-framework v1.32.0 h1:6emeE/YuMNmdd1pZ3NlJGu94QCa+NUayx26NX0jAQDY=
-github.com/smartcontractkit/chainlink-testing-framework v1.32.0/go.mod h1:wSRZGoukZliwfTkghdF4cI1RLHz5k3aDnL+rXhJ6f7k=
+github.com/smartcontractkit/chainlink-testing-framework v1.32.2 h1:j5OQ9Xt2aqAGJfpEkh1pxvOiZMVbxypjFEBZkdIEfZg=
+github.com/smartcontractkit/chainlink-testing-framework v1.32.2/go.mod h1:OYlILtWsQskqZgzIjRe7PpIEySJVhb4qMnGxOuzrUgA=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 h1:Kk5OVlx/5g9q3Z3lhxytZS4/f8ds1MiNM8yaHgK3Oe8=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU=
 github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo=

integration-tests/testconfig/testconfig.go

@@ -382,7 +382,7 @@ func GetConfig(configurationNames []string, product Product) (TestConfig, error)
 	}
 
 	logger.Info().Msg("Loading config values from default ~/.testsecrets env file")
-	err = ctf_config.LoadSecretEnvsFromFile()
+	err = ctf_config.LoadSecretEnvsFromFiles()
 	if err != nil {
 		return TestConfig{}, errors.Wrapf(err, "error reading test config values from ~/.testsecrets file")
 	}

tools/bin/go_core_fuzz

@@ -6,6 +6,11 @@ SCRIPT_PATH=`dirname "$0"`; SCRIPT_PATH=`eval "cd \"$SCRIPT_PATH\" && pwd"`
 OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"}
 USE_TEE="${USE_TEE:-true}"
 
+# To allow reuse in CI from other repositories
+FUZZ_TOOL_PATH=${FUZZ_TOOL_PATH:-"./fuzz"}
+GO_MODULE_ROOT_PATH=${GO_MODULE_ROOT_PATH:-"./"}
+FUZZ_TIMEOUT=${FUZZ_TIMEOUT:-10m}
+
 echo "Failed fuzz tests and panics: ---------------------"
 echo ""
 use_tee() {
@@ -19,7 +24,7 @@ use_tee() {
 # the amount of --seconds here is subject to change based on how long the CI job takes in the future
 # as we add more fuzz tests, we should take into consideration increasing this timelapse, so we can have enough coverage.
 # We are timing out after ~10mins in case the tests hang. (Current CI duration is ~8m, modify if needed)
-cd ./fuzz && timeout 10m ./fuzz_all_native.py --ci --seconds 420 | use_tee $OUTPUT_FILE
+timeout "${FUZZ_TIMEOUT}" "${FUZZ_TOOL_PATH}"/fuzz_all_native.py --ci --seconds 420 --go_module_root "${GO_MODULE_ROOT_PATH}" | use_tee $OUTPUT_FILE
 EXITCODE=${PIPESTATUS[0]}
 
 # Assert no known sensitive strings present in test logger output

tools/bin/go_core_race_tests

@@ -4,7 +4,11 @@ OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"}
 USE_TEE="${USE_TEE:-true}"
 TIMEOUT="${TIMEOUT:-30s}"
 COUNT="${COUNT:-10}"
-GO_LDFLAGS=$(bash tools/bin/ldflags)
+
+# To allow reuse in CI from other repositories
+TOOLS_PATH=${TOOLS_PATH:-"./tools"}
+
+GO_LDFLAGS=$(bash ${TOOLS_PATH}/bin/ldflags)
 use_tee() {
   if [ "$USE_TEE" = "true" ]; then
     tee "$@"

tools/bin/go_core_tests

@@ -6,9 +6,12 @@ SCRIPT_PATH=`dirname "$0"`; SCRIPT_PATH=`eval "cd \"$SCRIPT_PATH\" && pwd"`
 OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"}
 USE_TEE="${USE_TEE:-true}"
 
+# To allow reuse in CI from other repositories
+TOOLS_PATH=${TOOLS_PATH:-"./tools"}
+
 echo "Failed tests and panics: ---------------------"
 echo ""
-GO_LDFLAGS=$(bash tools/bin/ldflags)
+GO_LDFLAGS=$(bash ${TOOLS_PATH}/bin/ldflags)
 use_tee() {
   if [ "$USE_TEE" = "true" ]; then
     tee "$@"