Merge remote-tracking branch 'origin/develop' into sqlutil-wrapdb-txs

.changeset/pretty-kangaroos-tell.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+Add check for valid semvar value for changeset file #internal

.github/scripts/check-changeset-tags.sh

@@ -29,6 +29,14 @@ if [[ ! -f "$CHANGESET_FILE_PATH" ]]; then
   exit 1
 fi
 
+changeset_content=$(sed -n '/^---$/,/^---$/{ /^---$/!p; }' $CHANGESET_FILE_PATH)
+semvar_value=$(echo "$changeset_content" | awk -F": " '/"chainlink"/ {print $2}')
+
+if [[ "$semvar_value" != "major" && "$semvar_value" != "minor" && "$semvar_value" != "patch" ]]; then
+  echo "Invalid changeset semvar value for 'chainlink'. Must be 'major', 'minor', or 'patch'."
+  exit 1
+fi
+
 while IFS= read -r line; do
   for tag in "${tags_list[@]}"; do
     if [[ "$line" == *"$tag"* ]]; then

.github/workflows/integration-tests.yml

@@ -563,12 +563,12 @@ jobs:
             pyroscope_env: ci-smoke-vrf-evm-simulated
           - name: vrfv2
             id: vrfv2
-            nodes: 4
+            nodes: 5
             os: ubuntu-latest
             pyroscope_env: ci-smoke-vrf2-evm-simulated
           - name: vrfv2plus
             id: vrfv2plus
-            nodes: 7
+            nodes: 8
             os: ubuntu-latest
             pyroscope_env: ci-smoke-vrf2plus-evm-simulated
           - name: forwarder_ocr

core/capabilities/registry_test.go

@@ -42,6 +42,7 @@ func TestRegistry(t *testing.T) {
 		capabilities.CapabilityTypeAction,
 		"capability-1-description",
 		"v1.0.0",
+		nil,
 	)
 	require.NoError(t, err)
 
@@ -70,6 +71,7 @@ func TestRegistry_NoDuplicateIDs(t *testing.T) {
 		capabilities.CapabilityTypeAction,
 		"capability-1-description",
 		"v1.0.0",
+		nil,
 	)
 	require.NoError(t, err)
 
@@ -82,6 +84,7 @@ func TestRegistry_NoDuplicateIDs(t *testing.T) {
 		capabilities.CapabilityTypeConsensus,
 		"capability-2-description",
 		"v1.0.0",
+		nil,
 	)
 	require.NoError(t, err)
 	c2 := &mockCapability{CapabilityInfo: ci}
@@ -106,6 +109,7 @@ func TestRegistry_ChecksExecutionAPIByType(t *testing.T) {
 					capabilities.CapabilityTypeAction,
 					"capability-1-description",
 					"v1.0.0",
+					nil,
 				)
 				require.NoError(t, err)
 
@@ -126,6 +130,7 @@ func TestRegistry_ChecksExecutionAPIByType(t *testing.T) {
 					capabilities.CapabilityTypeTarget,
 					"capability-1-description",
 					"v1.0.0",
+					nil,
 				)
 				require.NoError(t, err)
 
@@ -159,6 +164,7 @@ func TestRegistry_ChecksExecutionAPIByType(t *testing.T) {
 					capabilities.CapabilityTypeConsensus,
 					"capability-1-description",
 					"v1.0.0",
+					nil,
 				)
 				require.NoError(t, err)
 

core/capabilities/targets/write_target.go

@@ -63,6 +63,7 @@ func NewEvmWrite(chain legacyevm.Chain, lggr logger.Logger) *EvmWrite {
 		capabilities.CapabilityTypeTarget,
 		"Write target.",
 		"v1.0.0",
+		nil,
 	)
 
 	return &EvmWrite{

core/scripts/go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/prometheus/client_golang v1.17.0
 	github.com/shopspring/decimal v1.3.1
 	github.com/smartcontractkit/chainlink-automation v1.0.3
-	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240424104752-ed1756cf454c
+	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240424132620-add4946c1c73
 	github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772
 	github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000
 	github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c

core/scripts/go.sum

@@ -1185,8 +1185,8 @@ github.com/smartcontractkit/chain-selectors v1.0.10 h1:t9kJeE6B6G+hKD0GYR4kGJSCq
 github.com/smartcontractkit/chain-selectors v1.0.10/go.mod h1:d4Hi+E1zqjy9HqMkjBE5q1vcG9VGgxf5VxiRHfzi2kE=
 github.com/smartcontractkit/chainlink-automation v1.0.3 h1:h/ijT0NiyV06VxYVgcNfsE3+8OEzT3Q0Z9au0z1BPWs=
 github.com/smartcontractkit/chainlink-automation v1.0.3/go.mod h1:RjboV0Qd7YP+To+OrzHGXaxUxoSONveCoAK2TQ1INLU=
-github.com/smartcontractkit/chainlink-common v0.1.7-0.20240424104752-ed1756cf454c h1:nk3g1il/cG0raV2ymNlytAPvjfYNSvwHP7Gfy6ItmSI=
-github.com/smartcontractkit/chainlink-common v0.1.7-0.20240424104752-ed1756cf454c/go.mod h1:GTDBbovHUSAUk+fuGIySF2A/whhdtHGaWmU61BoERks=
+github.com/smartcontractkit/chainlink-common v0.1.7-0.20240424132620-add4946c1c73 h1:54hM3/SrOM166it2K35hGb5K7gQ49/Op0aHp9WkqpqU=
+github.com/smartcontractkit/chainlink-common v0.1.7-0.20240424132620-add4946c1c73/go.mod h1:GTDBbovHUSAUk+fuGIySF2A/whhdtHGaWmU61BoERks=
 github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240419213354-ea34a948e2ee h1:eFuBKyEbL2b+eyfgV/Eu9+8HuCEev+IcBi+K9l1dG7g=
 github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240419213354-ea34a948e2ee/go.mod h1:uATrrJ8IsuBkOBJ46USuf73gz9gZy5k5bzGE5/ji/rc=
 github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240220203239-09be0ea34540 h1:xFSv8561jsLtF6gYZr/zW2z5qUUAkcFkApin2mnbYTo=

core/services/workflows/engine_test.go

@@ -136,6 +136,7 @@ func TestEngineWithHardcodedWorkflow(t *testing.T) {
 			capabilities.CapabilityTypeTarget,
 			"a write capability targeting ethereum sepolia testnet",
 			"v1.0.0",
+			nil,
 		),
 		func(req capabilities.CapabilityRequest) (capabilities.CapabilityResponse, error) {
 			m := req.Inputs.Underlying["report"].(*values.Map)
@@ -219,6 +220,7 @@ func mockTrigger(t *testing.T) (capabilities.TriggerCapability, capabilities.Cap
 			capabilities.CapabilityTypeTrigger,
 			"issues a trigger when a mercury report is received.",
 			"v1.0.0",
+			nil,
 		),
 		ch: make(chan capabilities.CapabilityResponse, 10),
 	}
@@ -242,6 +244,7 @@ func mockFailingConsensus() *mockCapability {
 			capabilities.CapabilityTypeConsensus,
 			"an ocr3 consensus capability",
 			"v3.0.0",
+			nil,
 		),
 		func(req capabilities.CapabilityRequest) (capabilities.CapabilityResponse, error) {
 			return capabilities.CapabilityResponse{}, errors.New("fatal consensus error")
@@ -256,6 +259,7 @@ func mockConsensus() *mockCapability {
 			capabilities.CapabilityTypeConsensus,
 			"an ocr3 consensus capability",
 			"v3.0.0",
+			nil,
 		),
 		func(req capabilities.CapabilityRequest) (capabilities.CapabilityResponse, error) {
 			obs := req.Inputs.Underlying["observations"]
@@ -282,6 +286,7 @@ func mockTarget() *mockCapability {
 			capabilities.CapabilityTypeTarget,
 			"a write capability targeting polygon mumbai testnet",
 			"v1.0.0",
+			nil,
 		),
 		func(req capabilities.CapabilityRequest) (capabilities.CapabilityResponse, error) {
 			m := req.Inputs.Underlying["report"].(*values.Map)
@@ -383,6 +388,7 @@ func mockAction() (*mockCapability, values.Value) {
 			capabilities.CapabilityTypeAction,
 			"a read chain action",
 			"v1.0.0",
+			nil,
 		),
 		func(req capabilities.CapabilityRequest) (capabilities.CapabilityResponse, error) {
 

crib/devspace.yaml

@@ -73,6 +73,8 @@ images:
         MACOS_SDK_DIR=$(pwd)/tools/bin/MacOSX12.3.sdk IMAGE=$image ./tools/bin/goreleaser_wrapper release --snapshot --clean --config .goreleaser.devspace.yaml
         docker push $image
 hooks:
+  - command: ./scripts/check_env_vars.sh
+    events: [ "before:deploy:app" ]
   - wait:
       running: true
       terminatedWithCode: 0

crib/scripts/check_env_vars.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# List of required environment variables
+required_vars=(
+  "DEVSPACE_IMAGE"
+  "DEVSPACE_INGRESS_CIDRS"
+  "DEVSPACE_INGRESS_BASE_DOMAIN"
+  "DEVSPACE_INGRESS_CERT_ARN"
+  "DEVSPACE_K8S_POD_WAIT_TIMEOUT"
+  "NS_TTL"
+  )
+
+missing_vars=0  # Counter for missing variables
+
+# Check each variable
+for var in "${required_vars[@]}"; do
+  if [ -z "${!var}" ]; then  # If variable is unset or empty
+    echo "Error: Environment variable $var is not set."
+    missing_vars=$((missing_vars + 1))
+  fi
+done
+
+# Exit with an error if any variables were missing
+if [ $missing_vars -ne 0 ]; then
+  echo "Total missing environment variables: $missing_vars"
+  echo "To fix it, add missing variables in the \"crib/.env\" file."
+  echo "you can find example of the .env config in the \"crib/.env.example\""
+  exit 1
+else
+  echo "All required environment variables are set."
+fi

go.mod

@@ -72,7 +72,7 @@ require (
 	github.com/shopspring/decimal v1.3.1
 	github.com/smartcontractkit/chain-selectors v1.0.10
 	github.com/smartcontractkit/chainlink-automation v1.0.3
-	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240424104752-ed1756cf454c
+	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240424132620-add4946c1c73
 	github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240419213354-ea34a948e2ee
 	github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240220203239-09be0ea34540
 	github.com/smartcontractkit/chainlink-feeds v0.0.0-20240422130241-13c17a91b2ab

go.sum

@@ -1180,8 +1180,8 @@ github.com/smartcontractkit/chain-selectors v1.0.10 h1:t9kJeE6B6G+hKD0GYR4kGJSCq
 github.com/smartcontractkit/chain-selectors v1.0.10/go.mod h1:d4Hi+E1zqjy9HqMkjBE5q1vcG9VGgxf5VxiRHfzi2kE=
 github.com/smartcontractkit/chainlink-automation v1.0.3 h1:h/ijT0NiyV06VxYVgcNfsE3+8OEzT3Q0Z9au0z1BPWs=
 github.com/smartcontractkit/chainlink-automation v1.0.3/go.mod h1:RjboV0Qd7YP+To+OrzHGXaxUxoSONveCoAK2TQ1INLU=
-github.com/smartcontractkit/chainlink-common v0.1.7-0.20240424104752-ed1756cf454c h1:nk3g1il/cG0raV2ymNlytAPvjfYNSvwHP7Gfy6ItmSI=
-github.com/smartcontractkit/chainlink-common v0.1.7-0.20240424104752-ed1756cf454c/go.mod h1:GTDBbovHUSAUk+fuGIySF2A/whhdtHGaWmU61BoERks=
+github.com/smartcontractkit/chainlink-common v0.1.7-0.20240424132620-add4946c1c73 h1:54hM3/SrOM166it2K35hGb5K7gQ49/Op0aHp9WkqpqU=
+github.com/smartcontractkit/chainlink-common v0.1.7-0.20240424132620-add4946c1c73/go.mod h1:GTDBbovHUSAUk+fuGIySF2A/whhdtHGaWmU61BoERks=
 github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240419213354-ea34a948e2ee h1:eFuBKyEbL2b+eyfgV/Eu9+8HuCEev+IcBi+K9l1dG7g=
 github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240419213354-ea34a948e2ee/go.mod h1:uATrrJ8IsuBkOBJ46USuf73gz9gZy5k5bzGE5/ji/rc=
 github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240220203239-09be0ea34540 h1:xFSv8561jsLtF6gYZr/zW2z5qUUAkcFkApin2mnbYTo=

integration-tests/actions/actions.go

@@ -8,14 +8,18 @@ import (
 	"errors"
 	"fmt"
 	"math/big"
+	"os"
 	"strings"
 	"sync"
 	"testing"
 	"time"
 
 	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/go-resty/resty/v2"
+	"github.com/rs/zerolog"
 
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
@@ -33,6 +37,7 @@ import (
 	"github.com/smartcontractkit/chainlink-testing-framework/utils/testcontext"
 	"github.com/smartcontractkit/chainlink/integration-tests/client"
 	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
+	"github.com/smartcontractkit/chainlink/integration-tests/docker/test_env"
 )
 
 // ContractDeploymentInterval After how many contract actions to wait before starting any more
@@ -562,3 +567,100 @@ func WaitForBlockNumberToBe(
 		}
 	}
 }
+
+// todo - move to EVMClient
+func RewindSimulatedChainToBlockNumber(
+	ctx context.Context,
+	evmClient blockchain.EVMClient,
+	rpcURL string,
+	rewindChainToBlockNumber uint64,
+	l zerolog.Logger,
+) (uint64, error) {
+	latestBlockNumberBeforeReorg, err := evmClient.LatestBlockNumber(ctx)
+	if err != nil {
+		return 0, fmt.Errorf("error getting latest block number: %w", err)
+	}
+
+	l.Info().
+		Str("RPC URL", rpcURL).
+		Uint64("Latest Block Number before Reorg", latestBlockNumberBeforeReorg).
+		Uint64("Rewind Chain to Block Number", rewindChainToBlockNumber).
+		Msg("Performing Reorg on chain by rewinding chain to specific block number")
+
+	_, err = NewRPCRawClient(rpcURL).SetHeadForSimulatedChain(rewindChainToBlockNumber)
+
+	if err != nil {
+		return 0, fmt.Errorf("error making reorg: %w", err)
+	}
+
+	err = evmClient.WaitForEvents()
+	if err != nil {
+		return 0, fmt.Errorf("error waiting for events: %w", err)
+	}
+
+	latestBlockNumberAfterReorg, err := evmClient.LatestBlockNumber(ctx)
+	if err != nil {
+		return 0, fmt.Errorf("error getting latest block number: %w", err)
+	}
+
+	l.Info().
+		Uint64("Block Number", latestBlockNumberAfterReorg).
+		Msg("Latest Block Number after Reorg")
+	return latestBlockNumberAfterReorg, nil
+}
+
+func GetRPCUrl(env *test_env.CLClusterTestEnv, chainID int64) (string, error) {
+	provider, err := env.GetRpcProvider(chainID)
+	if err != nil {
+		return "", err
+	}
+	return provider.PublicHttpUrls()[0], nil
+}
+
+// RPCRawClient
+// created separate client since method evmClient.RawJsonRPCCall fails on "invalid argument 0: json: cannot unmarshal non-string into Go value of type hexutil.Uint64"
+type RPCRawClient struct {
+	resty *resty.Client
+}
+
+func NewRPCRawClient(url string) *RPCRawClient {
+	isDebug := os.Getenv("DEBUG_RESTY") == "true"
+	restyClient := resty.New().SetDebug(isDebug).SetBaseURL(url)
+	return &RPCRawClient{
+		resty: restyClient,
+	}
+}
+
+func (g *RPCRawClient) SetHeadForSimulatedChain(setHeadToBlockNumber uint64) (JsonRPCResponse, error) {
+	var responseObject JsonRPCResponse
+	postBody, _ := json.Marshal(map[string]any{
+		"jsonrpc": "2.0",
+		"id":      1,
+		"method":  "debug_setHead",
+		"params":  []string{hexutil.EncodeUint64(setHeadToBlockNumber)},
+	})
+	resp, err := g.resty.R().
+		SetHeader("Content-Type", "application/json").
+		SetBody(postBody).
+		SetResult(&responseObject).
+		Post("")
+
+	if err != nil {
+		return JsonRPCResponse{}, fmt.Errorf("error making API request: %w", err)
+	}
+	statusCode := resp.StatusCode()
+	if statusCode != 200 && statusCode != 201 {
+		return JsonRPCResponse{}, fmt.Errorf("error invoking debug_setHead method, received unexpected status code %d: %s", statusCode, resp.String())
+	}
+	if responseObject.Error != "" {
+		return JsonRPCResponse{}, fmt.Errorf("received non-empty error field: %v", responseObject.Error)
+	}
+	return responseObject, nil
+}
+
+type JsonRPCResponse struct {
+	Version string `json:"jsonrpc"`
+	Id      int    `json:"id"`
+	Result  string `json:"result,omitempty"`
+	Error   string `json:"error,omitempty"`
+}

integration-tests/actions/vrf/vrfv2/contract_steps.go

@@ -506,7 +506,7 @@ func RequestRandomnessAndWaitForFulfillment(
 	randomnessRequestCountPerRequest uint16,
 	randomnessRequestCountPerRequestDeviation uint16,
 	randomWordsFulfilledEventTimeout time.Duration,
-) (*contracts.CoordinatorRandomWordsFulfilled, error) {
+) (*contracts.CoordinatorRandomWordsRequested, *contracts.CoordinatorRandomWordsFulfilled, error) {
 	randomWordsRequestedEvent, err := RequestRandomness(
 		l,
 		consumer,
@@ -520,18 +520,18 @@ func RequestRandomnessAndWaitForFulfillment(
 		randomnessRequestCountPerRequestDeviation,
 	)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	fulfillmentEvents, err := WaitRandomWordsFulfilledEvent(
+	randomWordsFulfilledEvent, err := WaitRandomWordsFulfilledEvent(
 		coordinator,
 		randomWordsRequestedEvent.RequestId,
 		randomWordsFulfilledEventTimeout,
 		l,
 	)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	return fulfillmentEvents, nil
+	return randomWordsRequestedEvent, randomWordsFulfilledEvent, nil
 }
 
 func RequestRandomness(