From 9ef44351f68de4c330ca95821b7a0359cc56e0fb Mon Sep 17 00:00:00 2001 From: Joe Huang Date: Mon, 30 Sep 2024 10:19:35 -0500 Subject: [PATCH 01/10] fixing multinode state transition logic by register polling subscription in rpc client (#14534) * polling subscription to be registered * adding changeset * fix Subscribe new head * add test * update changeset * fix lint * fix deadlock and add unit test for http polling sub * update changeset * adding sub closed check and remove import * add unit test coverage for http polling subscribeToHeads * update test * address comments part 1 * clean * part 2 * fix lint * fix lint (cherry picked from commit de268e98b8d68a284e1260297925b91c5d2411bc) --- .changeset/moody-rules-agree.md | 8 ++ core/chains/evm/client/rpc_client.go | 47 +++++++- core/chains/evm/client/rpc_client_test.go | 127 ++++++++++++++++++++++ 3 files changed, 177 insertions(+), 5 deletions(-) create mode 100644 .changeset/moody-rules-agree.md diff --git a/.changeset/moody-rules-agree.md b/.changeset/moody-rules-agree.md new file mode 100644 index 0000000000..ef1f3bcaf6 --- /dev/null +++ b/.changeset/moody-rules-agree.md @@ -0,0 +1,8 @@ +--- +"chainlink": patch +--- + +- register polling subscription to avoid subscription leaking when rpc client gets closed. +- add a temporary special treatment for SubscribeNewHead before we replace it with SubscribeToHeads. Add a goroutine that forwards new head from poller to caller channel. +- fix a deadlock in poller, by using a new lock for subs slice in rpc client. +#bugfix diff --git a/core/chains/evm/client/rpc_client.go b/core/chains/evm/client/rpc_client.go index 763348173a..a29ed5e118 100644 --- a/core/chains/evm/client/rpc_client.go +++ b/core/chains/evm/client/rpc_client.go @@ -129,7 +129,8 @@ type rpcClient struct { ws rawclient http *rawclient - stateMu sync.RWMutex // protects state* fields + stateMu sync.RWMutex // protects state* fields + subsSliceMu sync.RWMutex // protects subscription slice // Need to track subscriptions because closing the RPC does not (always?) // close the underlying subscription @@ -317,8 +318,8 @@ func (r *rpcClient) getRPCDomain() string { // registerSub adds the sub to the rpcClient list func (r *rpcClient) registerSub(sub ethereum.Subscription, stopInFLightCh chan struct{}) error { - r.stateMu.Lock() - defer r.stateMu.Unlock() + r.subsSliceMu.Lock() + defer r.subsSliceMu.Unlock() // ensure that the `sub` belongs to current life cycle of the `rpcClient` and it should not be killed due to // previous `DisconnectAll` call. select { @@ -335,12 +336,16 @@ func (r *rpcClient) registerSub(sub ethereum.Subscription, stopInFLightCh chan s // DisconnectAll disconnects all clients connected to the rpcClient func (r *rpcClient) DisconnectAll() { r.stateMu.Lock() - defer r.stateMu.Unlock() if r.ws.rpc != nil { r.ws.rpc.Close() } r.cancelInflightRequests() + r.stateMu.Unlock() + + r.subsSliceMu.Lock() r.unsubscribeAll() + r.subsSliceMu.Unlock() + r.chainInfoLock.Lock() r.latestChainInfo = commonclient.ChainInfo{} r.chainInfoLock.Unlock() @@ -496,11 +501,30 @@ func (r *rpcClient) SubscribeNewHead(ctx context.Context, channel chan<- *evmtyp if r.newHeadsPollInterval > 0 { interval := r.newHeadsPollInterval timeout := interval - poller, _ := commonclient.NewPoller[*evmtypes.Head](interval, r.latestBlock, timeout, r.rpcLog) + poller, pollerCh := commonclient.NewPoller[*evmtypes.Head](interval, r.latestBlock, timeout, r.rpcLog) if err = poller.Start(ctx); err != nil { return nil, err } + // NOTE this is a temporary special treatment for SubscribeNewHead before we refactor head tracker to use SubscribeToHeads + // as we need to forward new head from the poller channel to the channel passed from caller. + go func() { + for head := range pollerCh { + select { + case channel <- head: + // forwarding new head to the channel passed from caller + case <-poller.Err(): + // return as poller returns error + return + } + } + }() + + err = r.registerSub(&poller, chStopInFlight) + if err != nil { + return nil, err + } + lggr.Debugf("Polling new heads over http") return &poller, nil } @@ -547,6 +571,11 @@ func (r *rpcClient) SubscribeToHeads(ctx context.Context) (ch <-chan *evmtypes.H return nil, nil, err } + err = r.registerSub(&poller, chStopInFlight) + if err != nil { + return nil, nil, err + } + lggr.Debugf("Polling new heads over http") return channel, &poller, nil } @@ -579,6 +608,8 @@ func (r *rpcClient) SubscribeToHeads(ctx context.Context) (ch <-chan *evmtypes.H } func (r *rpcClient) SubscribeToFinalizedHeads(ctx context.Context) (<-chan *evmtypes.Head, commontypes.Subscription, error) { + ctx, cancel, chStopInFlight, _, _ := r.acquireQueryCtx(ctx, r.rpcTimeout) + defer cancel() interval := r.finalizedBlockPollInterval if interval == 0 { return nil, nil, errors.New("FinalizedBlockPollInterval is 0") @@ -588,6 +619,12 @@ func (r *rpcClient) SubscribeToFinalizedHeads(ctx context.Context) (<-chan *evmt if err := poller.Start(ctx); err != nil { return nil, nil, err } + + err := r.registerSub(&poller, chStopInFlight) + if err != nil { + return nil, nil, err + } + return channel, &poller, nil } diff --git a/core/chains/evm/client/rpc_client_test.go b/core/chains/evm/client/rpc_client_test.go index b594a0ca16..d959f8d111 100644 --- a/core/chains/evm/client/rpc_client_test.go +++ b/core/chains/evm/client/rpc_client_test.go @@ -19,6 +19,8 @@ import ( "github.com/tidwall/gjson" "go.uber.org/zap" + commontypes "github.com/smartcontractkit/chainlink/v2/common/types" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "github.com/smartcontractkit/chainlink-common/pkg/logger" @@ -57,6 +59,25 @@ func TestRPCClient_SubscribeNewHead(t *testing.T) { } return } + + checkClosedRPCClientShouldRemoveExistingSub := func(t tests.TestingT, ctx context.Context, sub commontypes.Subscription, rpcClient client.RPCClient) { + errCh := sub.Err() + + // ensure sub exists + require.Equal(t, int32(1), rpcClient.SubscribersCount()) + rpcClient.DisconnectAll() + + // ensure sub is closed + select { + case <-errCh: // ok + default: + assert.Fail(t, "channel should be closed") + } + + require.NoError(t, rpcClient.Dial(ctx)) + require.Equal(t, int32(0), rpcClient.SubscribersCount()) + } + t.Run("Updates chain info on new blocks", func(t *testing.T) { server := testutils.NewWSServer(t, chainId, serverCallBack) wsURL := server.WSURL() @@ -131,6 +152,50 @@ func TestRPCClient_SubscribeNewHead(t *testing.T) { assert.Equal(t, int64(0), highestUserObservations.FinalizedBlockNumber) assert.Equal(t, (*big.Int)(nil), highestUserObservations.TotalDifficulty) }) + t.Run("SubscribeToHeads with http polling enabled will update new heads", func(t *testing.T) { + type rpcServer struct { + Head *evmtypes.Head + URL *url.URL + } + createRPCServer := func() *rpcServer { + server := &rpcServer{} + server.Head = &evmtypes.Head{Number: 127} + server.URL = testutils.NewWSServer(t, chainId, func(method string, params gjson.Result) (resp testutils.JSONRPCResponse) { + assert.Equal(t, "eth_getBlockByNumber", method) + if assert.True(t, params.IsArray()) && assert.Equal(t, "latest", params.Array()[0].String()) { + head := server.Head + jsonHead, err := json.Marshal(head) + if err != nil { + panic(fmt.Errorf("failed to marshal head: %w", err)) + } + resp.Result = string(jsonHead) + } + + return + }).WSURL() + return server + } + + server := createRPCServer() + rpc := client.NewRPCClient(lggr, *server.URL, nil, "rpc", 1, chainId, commonclient.Primary, 0, tests.TestInterval, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + defer rpc.Close() + require.NoError(t, rpc.Dial(ctx)) + latest, highestUserObservations := rpc.GetInterceptedChainInfo() + // latest chain info hasn't been initialized + assert.Equal(t, int64(0), latest.BlockNumber) + assert.Equal(t, int64(0), highestUserObservations.BlockNumber) + + headCh, sub, err := rpc.SubscribeToHeads(commonclient.CtxAddHealthCheckFlag(tests.Context(t))) + require.NoError(t, err) + defer sub.Unsubscribe() + + head := <-headCh + assert.Equal(t, server.Head.Number, head.BlockNumber()) + // the http polling subscription should update the head block + latest, highestUserObservations = rpc.GetInterceptedChainInfo() + assert.Equal(t, server.Head.Number, latest.BlockNumber) + assert.Equal(t, server.Head.Number, highestUserObservations.BlockNumber) + }) t.Run("Concurrent Unsubscribe and onNewHead calls do not lead to a deadlock", func(t *testing.T) { const numberOfAttempts = 1000 // need a large number to increase the odds of reproducing the issue server := testutils.NewWSServer(t, chainId, serverCallBack) @@ -184,6 +249,68 @@ func TestRPCClient_SubscribeNewHead(t *testing.T) { require.ErrorContains(t, err, "RPCClient returned error (rpc)") tests.AssertLogEventually(t, observed, "evmclient.Client#EthSubscribe RPC call failure") }) + t.Run("Closed rpc client should remove existing SubscribeNewHead subscription with WS", func(t *testing.T) { + server := testutils.NewWSServer(t, chainId, serverCallBack) + wsURL := server.WSURL() + + rpc := client.NewRPCClient(lggr, *wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + defer rpc.Close() + require.NoError(t, rpc.Dial(ctx)) + + ch := make(chan *evmtypes.Head) + sub, err := rpc.SubscribeNewHead(tests.Context(t), ch) + require.NoError(t, err) + checkClosedRPCClientShouldRemoveExistingSub(t, ctx, sub, rpc) + }) + t.Run("Closed rpc client should remove existing SubscribeNewHead subscription with HTTP polling", func(t *testing.T) { + server := testutils.NewWSServer(t, chainId, serverCallBack) + wsURL := server.WSURL() + + rpc := client.NewRPCClient(lggr, *wsURL, &url.URL{}, "rpc", 1, chainId, commonclient.Primary, 0, 1, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + defer rpc.Close() + require.NoError(t, rpc.Dial(ctx)) + + ch := make(chan *evmtypes.Head) + sub, err := rpc.SubscribeNewHead(tests.Context(t), ch) + require.NoError(t, err) + checkClosedRPCClientShouldRemoveExistingSub(t, ctx, sub, rpc) + }) + t.Run("Closed rpc client should remove existing SubscribeToHeads subscription with WS", func(t *testing.T) { + server := testutils.NewWSServer(t, chainId, serverCallBack) + wsURL := server.WSURL() + + rpc := client.NewRPCClient(lggr, *wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + defer rpc.Close() + require.NoError(t, rpc.Dial(ctx)) + + _, sub, err := rpc.SubscribeToHeads(tests.Context(t)) + require.NoError(t, err) + checkClosedRPCClientShouldRemoveExistingSub(t, ctx, sub, rpc) + }) + t.Run("Closed rpc client should remove existing SubscribeToHeads subscription with HTTP polling", func(t *testing.T) { + server := testutils.NewWSServer(t, chainId, serverCallBack) + wsURL := server.WSURL() + + rpc := client.NewRPCClient(lggr, *wsURL, &url.URL{}, "rpc", 1, chainId, commonclient.Primary, 0, 1, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + defer rpc.Close() + require.NoError(t, rpc.Dial(ctx)) + + _, sub, err := rpc.SubscribeToHeads(tests.Context(t)) + require.NoError(t, err) + checkClosedRPCClientShouldRemoveExistingSub(t, ctx, sub, rpc) + }) + t.Run("Closed rpc client should remove existing SubscribeToFinalizedHeads subscription", func(t *testing.T) { + server := testutils.NewWSServer(t, chainId, serverCallBack) + wsURL := server.WSURL() + + rpc := client.NewRPCClient(lggr, *wsURL, &url.URL{}, "rpc", 1, chainId, commonclient.Primary, 1, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + defer rpc.Close() + require.NoError(t, rpc.Dial(ctx)) + + _, sub, err := rpc.SubscribeToFinalizedHeads(tests.Context(t)) + require.NoError(t, err) + checkClosedRPCClientShouldRemoveExistingSub(t, ctx, sub, rpc) + }) t.Run("Subscription error is properly wrapper", func(t *testing.T) { server := testutils.NewWSServer(t, chainId, serverCallBack) wsURL := server.WSURL() From cdc9d77813e8d08ad15d5b41973e8ee0f33437cd Mon Sep 17 00:00:00 2001 From: Joe Huang Date: Thu, 3 Oct 2024 10:53:09 -0400 Subject: [PATCH 02/10] WS URL can be optional when LogBroadcaster is disabled (#14364) * WS URL can be optional * add changeset * change * make WSURL optional * fix test, and enforce SubscribeFilterLogs to fail when ws url not provided * add comments * update changeset * update dial logic and make ws optional not required * fix test * fix * fix lint * address comments * update comments * fix test * add check when both ws and http missing * add test and add restrictions * add comment * revert outdated change * remove extra line * fix test * revert changes from rpc client * unintended change * remove unused * update verification logic * add test fix * modify unit test to cover logbroadcaster enabled false * update doc * udpate changeset * address PR comments * address pr comments * update invalid toml config * fix test * ws required for primary nodes when logbroadcaster enabled * minor * Dmytro's comments * fix nil ptr, more fix to come * fix make * refactor function sig * fix test * fix * make ws pointer * fix * fix make * address comments * fix lint * fix make * fix make * fix make * fix rpc disconnect with optional ws url --------- Co-authored-by: Dmytro Haidashenko (cherry picked from commit 5d96be59a27f68f2f491a7d9f8cb0b2af4e0e835) --- .changeset/silly-lies-boil.md | 8 ++ common/client/node.go | 17 +++-- common/client/node_test.go | 4 +- core/chains/evm/client/config_builder.go | 18 +++-- core/chains/evm/client/config_builder_test.go | 4 +- core/chains/evm/client/evm_client.go | 11 ++- core/chains/evm/client/helpers_test.go | 10 +-- core/chains/evm/client/rpc_client.go | 74 ++++++++++++------- core/chains/evm/client/rpc_client_test.go | 50 ++++++++----- core/chains/evm/config/toml/config.go | 36 ++++----- core/config/docs/chains-evm.toml | 2 +- core/services/chainlink/config_test.go | 18 ++--- docs/CONFIG.md | 2 +- 13 files changed, 157 insertions(+), 97 deletions(-) create mode 100644 .changeset/silly-lies-boil.md diff --git a/.changeset/silly-lies-boil.md b/.changeset/silly-lies-boil.md new file mode 100644 index 0000000000..b2a5084a36 --- /dev/null +++ b/.changeset/silly-lies-boil.md @@ -0,0 +1,8 @@ +--- +"chainlink": minor +--- + +Make websocket URL `WSURL` for `EVM.Nodes` optional, and apply logic so that: +* If WS URL was not provided, SubscribeFilterLogs should fail with an explicit error +* If WS URL was not provided LogBroadcaster should be disabled +#nops diff --git a/common/client/node.go b/common/client/node.go index 1f55e69cac..7885fe7676 100644 --- a/common/client/node.go +++ b/common/client/node.go @@ -91,14 +91,14 @@ type node[ services.StateMachine lfcLog logger.Logger name string - id int32 + id int chainID CHAIN_ID nodePoolCfg NodeConfig chainCfg ChainConfig order int32 chainFamily string - ws url.URL + ws *url.URL http *url.URL rpc RPC @@ -121,10 +121,10 @@ func NewNode[ nodeCfg NodeConfig, chainCfg ChainConfig, lggr logger.Logger, - wsuri url.URL, + wsuri *url.URL, httpuri *url.URL, name string, - id int32, + id int, chainID CHAIN_ID, nodeOrder int32, rpc RPC, @@ -136,8 +136,10 @@ func NewNode[ n.chainID = chainID n.nodePoolCfg = nodeCfg n.chainCfg = chainCfg - n.ws = wsuri n.order = nodeOrder + if wsuri != nil { + n.ws = wsuri + } if httpuri != nil { n.http = httpuri } @@ -157,7 +159,10 @@ func NewNode[ } func (n *node[CHAIN_ID, HEAD, RPC]) String() string { - s := fmt.Sprintf("(%s)%s:%s", Primary.String(), n.name, n.ws.String()) + s := fmt.Sprintf("(%s)%s", Primary.String(), n.name) + if n.ws != nil { + s = s + fmt.Sprintf(":%s", n.ws.String()) + } if n.http != nil { s = s + fmt.Sprintf(":%s", n.http.String()) } diff --git a/common/client/node_test.go b/common/client/node_test.go index 66bb50fc94..539964691c 100644 --- a/common/client/node_test.go +++ b/common/client/node_test.go @@ -67,10 +67,10 @@ type testNodeOpts struct { config testNodeConfig chainConfig clientMocks.ChainConfig lggr logger.Logger - wsuri url.URL + wsuri *url.URL httpuri *url.URL name string - id int32 + id int chainID types.ID nodeOrder int32 rpc *mockNodeClient[types.ID, Head] diff --git a/core/chains/evm/client/config_builder.go b/core/chains/evm/client/config_builder.go index 9a31f9e4b4..66bdfc2614 100644 --- a/core/chains/evm/client/config_builder.go +++ b/core/chains/evm/client/config_builder.go @@ -80,15 +80,21 @@ func NewClientConfigs( func parseNodeConfigs(nodeCfgs []NodeConfig) ([]*toml.Node, error) { nodes := make([]*toml.Node, len(nodeCfgs)) for i, nodeCfg := range nodeCfgs { - if nodeCfg.WSURL == nil || nodeCfg.HTTPURL == nil { - return nil, fmt.Errorf("node config [%d]: missing WS or HTTP URL", i) + var wsURL, httpURL *commonconfig.URL + // wsUrl requirement will be checked in EVMConfig validation + if nodeCfg.WSURL != nil { + wsURL = commonconfig.MustParseURL(*nodeCfg.WSURL) } - wsUrl := commonconfig.MustParseURL(*nodeCfg.WSURL) - httpUrl := commonconfig.MustParseURL(*nodeCfg.HTTPURL) + + if nodeCfg.HTTPURL == nil { + return nil, fmt.Errorf("node config [%d]: missing HTTP URL", i) + } + + httpURL = commonconfig.MustParseURL(*nodeCfg.HTTPURL) node := &toml.Node{ Name: nodeCfg.Name, - WSURL: wsUrl, - HTTPURL: httpUrl, + WSURL: wsURL, + HTTPURL: httpURL, SendOnly: nodeCfg.SendOnly, Order: nodeCfg.Order, } diff --git a/core/chains/evm/client/config_builder_test.go b/core/chains/evm/client/config_builder_test.go index 0339131171..ac12a63fbe 100644 --- a/core/chains/evm/client/config_builder_test.go +++ b/core/chains/evm/client/config_builder_test.go @@ -90,7 +90,7 @@ func TestNodeConfigs(t *testing.T) { require.Len(t, tomlNodes, len(nodeConfigs)) }) - t.Run("parsing missing ws url fails", func(t *testing.T) { + t.Run("ws can be optional", func(t *testing.T) { nodeConfigs := []client.NodeConfig{ { Name: ptr("foo1"), @@ -98,7 +98,7 @@ func TestNodeConfigs(t *testing.T) { }, } _, err := client.ParseTestNodeConfigs(nodeConfigs) - require.Error(t, err) + require.Nil(t, err) }) t.Run("parsing missing http url fails", func(t *testing.T) { diff --git a/core/chains/evm/client/evm_client.go b/core/chains/evm/client/evm_client.go index c26362d635..c596bbc3a9 100644 --- a/core/chains/evm/client/evm_client.go +++ b/core/chains/evm/client/evm_client.go @@ -15,22 +15,25 @@ import ( ) func NewEvmClient(cfg evmconfig.NodePool, chainCfg commonclient.ChainConfig, clientErrors evmconfig.ClientErrors, lggr logger.Logger, chainID *big.Int, nodes []*toml.Node, chainType chaintype.ChainType) Client { - var empty url.URL var primaries []commonclient.Node[*big.Int, *evmtypes.Head, RPCClient] var sendonlys []commonclient.SendOnlyNode[*big.Int, RPCClient] largePayloadRPCTimeout, defaultRPCTimeout := getRPCTimeouts(chainType) for i, node := range nodes { + var ws *url.URL + if node.WSURL != nil { + ws = (*url.URL)(node.WSURL) + } if node.SendOnly != nil && *node.SendOnly { - rpc := NewRPCClient(lggr, empty, (*url.URL)(node.HTTPURL), *node.Name, int32(i), chainID, + rpc := NewRPCClient(lggr, nil, (*url.URL)(node.HTTPURL), *node.Name, i, chainID, commonclient.Secondary, cfg.FinalizedBlockPollInterval(), cfg.NewHeadsPollInterval(), largePayloadRPCTimeout, defaultRPCTimeout, chainType) sendonly := commonclient.NewSendOnlyNode(lggr, (url.URL)(*node.HTTPURL), *node.Name, chainID, rpc) sendonlys = append(sendonlys, sendonly) } else { - rpc := NewRPCClient(lggr, (url.URL)(*node.WSURL), (*url.URL)(node.HTTPURL), *node.Name, int32(i), + rpc := NewRPCClient(lggr, ws, (*url.URL)(node.HTTPURL), *node.Name, i, chainID, commonclient.Primary, cfg.FinalizedBlockPollInterval(), cfg.NewHeadsPollInterval(), largePayloadRPCTimeout, defaultRPCTimeout, chainType) primaryNode := commonclient.NewNode(cfg, chainCfg, - lggr, (url.URL)(*node.WSURL), (*url.URL)(node.HTTPURL), *node.Name, int32(i), chainID, *node.Order, + lggr, ws, (*url.URL)(node.HTTPURL), *node.Name, i, chainID, *node.Order, rpc, "EVM") primaries = append(primaries, primaryNode) } diff --git a/core/chains/evm/client/helpers_test.go b/core/chains/evm/client/helpers_test.go index 031f648157..1a6090e4a0 100644 --- a/core/chains/evm/client/helpers_test.go +++ b/core/chains/evm/client/helpers_test.go @@ -135,7 +135,7 @@ func NewChainClientWithTestNode( rpcUrl string, rpcHTTPURL *url.URL, sendonlyRPCURLs []url.URL, - id int32, + id int, chainID *big.Int, ) (Client, error) { parsed, err := url.ParseRequestURI(rpcUrl) @@ -148,10 +148,10 @@ func NewChainClientWithTestNode( } lggr := logger.Test(t) - rpc := NewRPCClient(lggr, *parsed, rpcHTTPURL, "eth-primary-rpc-0", id, chainID, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + rpc := NewRPCClient(lggr, parsed, rpcHTTPURL, "eth-primary-rpc-0", id, chainID, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") n := commonclient.NewNode[*big.Int, *evmtypes.Head, RPCClient]( - nodeCfg, clientMocks.ChainConfig{NoNewHeadsThresholdVal: noNewHeadsThreshold}, lggr, *parsed, rpcHTTPURL, "eth-primary-node-0", id, chainID, 1, rpc, "EVM") + nodeCfg, clientMocks.ChainConfig{NoNewHeadsThresholdVal: noNewHeadsThreshold}, lggr, parsed, rpcHTTPURL, "eth-primary-node-0", id, chainID, 1, rpc, "EVM") primaries := []commonclient.Node[*big.Int, *evmtypes.Head, RPCClient]{n} var sendonlys []commonclient.SendOnlyNode[*big.Int, RPCClient] @@ -160,7 +160,7 @@ func NewChainClientWithTestNode( return nil, pkgerrors.Errorf("sendonly ethereum rpc url scheme must be http(s): %s", u.String()) } var empty url.URL - rpc := NewRPCClient(lggr, empty, &sendonlyRPCURLs[i], fmt.Sprintf("eth-sendonly-rpc-%d", i), id, chainID, commonclient.Secondary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + rpc := NewRPCClient(lggr, &empty, &sendonlyRPCURLs[i], fmt.Sprintf("eth-sendonly-rpc-%d", i), id, chainID, commonclient.Secondary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") s := commonclient.NewSendOnlyNode[*big.Int, RPCClient]( lggr, u, fmt.Sprintf("eth-sendonly-%d", i), chainID, rpc) sendonlys = append(sendonlys, s) @@ -206,7 +206,7 @@ func NewChainClientWithMockedRpc( parsed, _ := url.ParseRequestURI("ws://test") n := commonclient.NewNode[*big.Int, *evmtypes.Head, RPCClient]( - cfg, clientMocks.ChainConfig{NoNewHeadsThresholdVal: noNewHeadsThreshold}, lggr, *parsed, nil, "eth-primary-node-0", 1, chainID, 1, rpc, "EVM") + cfg, clientMocks.ChainConfig{NoNewHeadsThresholdVal: noNewHeadsThreshold}, lggr, parsed, nil, "eth-primary-node-0", 1, chainID, 1, rpc, "EVM") primaries := []commonclient.Node[*big.Int, *evmtypes.Head, RPCClient]{n} clientErrors := NewTestClientErrors() c := NewChainClient(lggr, selectionMode, leaseDuration, noNewHeadsThreshold, primaries, nil, chainID, chainType, &clientErrors, 0) diff --git a/core/chains/evm/client/rpc_client.go b/core/chains/evm/client/rpc_client.go index a29ed5e118..f55c35980d 100644 --- a/core/chains/evm/client/rpc_client.go +++ b/core/chains/evm/client/rpc_client.go @@ -117,7 +117,7 @@ type rawclient struct { type rpcClient struct { rpcLog logger.SugaredLogger name string - id int32 + id int chainID *big.Int tier commonclient.NodeTier largePayloadRpcTimeout time.Duration @@ -126,7 +126,7 @@ type rpcClient struct { newHeadsPollInterval time.Duration chainType chaintype.ChainType - ws rawclient + ws *rawclient http *rawclient stateMu sync.RWMutex // protects state* fields @@ -154,10 +154,10 @@ type rpcClient struct { // NewRPCCLient returns a new *rpcClient as commonclient.RPC func NewRPCClient( lggr logger.Logger, - wsuri url.URL, + wsuri *url.URL, httpuri *url.URL, name string, - id int32, + id int, chainID *big.Int, tier commonclient.NodeTier, finalizedBlockPollInterval time.Duration, @@ -175,9 +175,11 @@ func NewRPCClient( r.id = id r.chainID = chainID r.tier = tier - r.ws.uri = wsuri r.finalizedBlockPollInterval = finalizedBlockPollInterval r.newHeadsPollInterval = newHeadsPollInterval + if wsuri != nil { + r.ws = &rawclient{uri: *wsuri} + } if httpuri != nil { r.http = &rawclient{uri: *httpuri} } @@ -199,30 +201,33 @@ func (r *rpcClient) Dial(callerCtx context.Context) error { ctx, cancel := r.makeQueryCtx(callerCtx, r.rpcTimeout) defer cancel() - promEVMPoolRPCNodeDials.WithLabelValues(r.chainID.String(), r.name).Inc() - lggr := r.rpcLog.With("wsuri", r.ws.uri.Redacted()) - if r.http != nil { - lggr = lggr.With("httpuri", r.http.uri.Redacted()) + if r.ws == nil && r.http == nil { + return errors.New("cannot dial rpc client when both ws and http info are missing") } - lggr.Debugw("RPC dial: evmclient.Client#dial") - wsrpc, err := rpc.DialWebsocket(ctx, r.ws.uri.String(), "") - if err != nil { - promEVMPoolRPCNodeDialsFailed.WithLabelValues(r.chainID.String(), r.name).Inc() - return r.wrapRPCClientError(pkgerrors.Wrapf(err, "error while dialing websocket: %v", r.ws.uri.Redacted())) - } + promEVMPoolRPCNodeDials.WithLabelValues(r.chainID.String(), r.name).Inc() + lggr := r.rpcLog + if r.ws != nil { + lggr = lggr.With("wsuri", r.ws.uri.Redacted()) + wsrpc, err := rpc.DialWebsocket(ctx, r.ws.uri.String(), "") + if err != nil { + promEVMPoolRPCNodeDialsFailed.WithLabelValues(r.chainID.String(), r.name).Inc() + return r.wrapRPCClientError(pkgerrors.Wrapf(err, "error while dialing websocket: %v", r.ws.uri.Redacted())) + } - r.ws.rpc = wsrpc - r.ws.geth = ethclient.NewClient(wsrpc) + r.ws.rpc = wsrpc + r.ws.geth = ethclient.NewClient(wsrpc) + } if r.http != nil { + lggr = lggr.With("httpuri", r.http.uri.Redacted()) if err := r.DialHTTP(); err != nil { return err } } + lggr.Debugw("RPC dial: evmclient.Client#dial") promEVMPoolRPCNodeDialsSuccess.WithLabelValues(r.chainID.String(), r.name).Inc() - return nil } @@ -231,7 +236,7 @@ func (r *rpcClient) Dial(callerCtx context.Context) error { // It can only return error if the URL is malformed. func (r *rpcClient) DialHTTP() error { promEVMPoolRPCNodeDials.WithLabelValues(r.chainID.String(), r.name).Inc() - lggr := r.rpcLog.With("httpuri", r.ws.uri.Redacted()) + lggr := r.rpcLog.With("httpuri", r.http.uri.Redacted()) lggr.Debugw("RPC dial: evmclient.Client#dial") var httprpc *rpc.Client @@ -251,7 +256,7 @@ func (r *rpcClient) DialHTTP() error { func (r *rpcClient) Close() { defer func() { - if r.ws.rpc != nil { + if r.ws != nil && r.ws.rpc != nil { r.ws.rpc.Close() } }() @@ -270,7 +275,10 @@ func (r *rpcClient) cancelInflightRequests() { } func (r *rpcClient) String() string { - s := fmt.Sprintf("(%s)%s:%s", r.tier.String(), r.name, r.ws.uri.Redacted()) + s := fmt.Sprintf("(%s)%s", r.tier.String(), r.name) + if r.ws != nil { + s = s + fmt.Sprintf(":%s", r.ws.uri.Redacted()) + } if r.http != nil { s = s + fmt.Sprintf(":%s", r.http.uri.Redacted()) } @@ -336,7 +344,7 @@ func (r *rpcClient) registerSub(sub ethereum.Subscription, stopInFLightCh chan s // DisconnectAll disconnects all clients connected to the rpcClient func (r *rpcClient) DisconnectAll() { r.stateMu.Lock() - if r.ws.rpc != nil { + if r.ws != nil && r.ws.rpc != nil { r.ws.rpc.Close() } r.cancelInflightRequests() @@ -497,7 +505,6 @@ func (r *rpcClient) SubscribeNewHead(ctx context.Context, channel chan<- *evmtyp defer cancel() args := []interface{}{"newHeads"} lggr := r.newRqLggr().With("args", args) - if r.newHeadsPollInterval > 0 { interval := r.newHeadsPollInterval timeout := interval @@ -529,6 +536,10 @@ func (r *rpcClient) SubscribeNewHead(ctx context.Context, channel chan<- *evmtyp return &poller, nil } + if ws == nil { + return nil, errors.New("SubscribeNewHead is not allowed without ws url") + } + lggr.Debug("RPC call: evmclient.Client#EthSubscribe") start := time.Now() defer func() { @@ -557,7 +568,6 @@ func (r *rpcClient) SubscribeNewHead(ctx context.Context, channel chan<- *evmtyp func (r *rpcClient) SubscribeToHeads(ctx context.Context) (ch <-chan *evmtypes.Head, sub commontypes.Subscription, err error) { ctx, cancel, chStopInFlight, ws, _ := r.acquireQueryCtx(ctx, r.rpcTimeout) defer cancel() - args := []interface{}{rpcSubscriptionMethodNewHeads} start := time.Now() lggr := r.newRqLggr().With("args", args) @@ -580,6 +590,10 @@ func (r *rpcClient) SubscribeToHeads(ctx context.Context) (ch <-chan *evmtypes.H return channel, &poller, nil } + if ws == nil { + return nil, nil, errors.New("SubscribeNewHead is not allowed without ws url") + } + lggr.Debug("RPC call: evmclient.Client#EthSubscribe") defer func() { duration := time.Since(start) @@ -1286,6 +1300,9 @@ func (r *rpcClient) ClientVersion(ctx context.Context) (version string, err erro func (r *rpcClient) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- types.Log) (_ ethereum.Subscription, err error) { ctx, cancel, chStopInFlight, ws, _ := r.acquireQueryCtx(ctx, r.rpcTimeout) defer cancel() + if ws == nil { + return nil, errors.New("SubscribeFilterLogs is not allowed without ws url") + } lggr := r.newRqLggr().With("q", q) lggr.Debug("RPC call: evmclient.Client#SubscribeFilterLogs") @@ -1390,18 +1407,21 @@ func (r *rpcClient) wrapHTTP(err error) error { } // makeLiveQueryCtxAndSafeGetClients wraps makeQueryCtx -func (r *rpcClient) makeLiveQueryCtxAndSafeGetClients(parentCtx context.Context, timeout time.Duration) (ctx context.Context, cancel context.CancelFunc, ws rawclient, http *rawclient) { +func (r *rpcClient) makeLiveQueryCtxAndSafeGetClients(parentCtx context.Context, timeout time.Duration) (ctx context.Context, cancel context.CancelFunc, ws *rawclient, http *rawclient) { ctx, cancel, _, ws, http = r.acquireQueryCtx(parentCtx, timeout) return } func (r *rpcClient) acquireQueryCtx(parentCtx context.Context, timeout time.Duration) (ctx context.Context, cancel context.CancelFunc, - chStopInFlight chan struct{}, ws rawclient, http *rawclient) { + chStopInFlight chan struct{}, ws *rawclient, http *rawclient) { // Need to wrap in mutex because state transition can cancel and replace the // context r.stateMu.RLock() chStopInFlight = r.chStopInFlight - ws = r.ws + if r.ws != nil { + cp := *r.ws + ws = &cp + } if r.http != nil { cp := *r.http http = &cp diff --git a/core/chains/evm/client/rpc_client_test.go b/core/chains/evm/client/rpc_client_test.go index d959f8d111..662c757ffb 100644 --- a/core/chains/evm/client/rpc_client_test.go +++ b/core/chains/evm/client/rpc_client_test.go @@ -78,11 +78,18 @@ func TestRPCClient_SubscribeNewHead(t *testing.T) { require.Equal(t, int32(0), rpcClient.SubscribersCount()) } + t.Run("WS and HTTP URL cannot be both empty", func(t *testing.T) { + // ws is optional when LogBroadcaster is disabled, however SubscribeFilterLogs will return error if ws is missing + observedLggr, _ := logger.TestObserved(t, zap.DebugLevel) + rpcClient := client.NewRPCClient(observedLggr, nil, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + require.Equal(t, errors.New("cannot dial rpc client when both ws and http info are missing"), rpcClient.Dial(ctx)) + }) + t.Run("Updates chain info on new blocks", func(t *testing.T) { server := testutils.NewWSServer(t, chainId, serverCallBack) wsURL := server.WSURL() - rpc := client.NewRPCClient(lggr, *wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + rpc := client.NewRPCClient(lggr, wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") defer rpc.Close() require.NoError(t, rpc.Dial(ctx)) // set to default values @@ -132,7 +139,7 @@ func TestRPCClient_SubscribeNewHead(t *testing.T) { server := testutils.NewWSServer(t, chainId, serverCallBack) wsURL := server.WSURL() - rpc := client.NewRPCClient(lggr, *wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + rpc := client.NewRPCClient(lggr, wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") defer rpc.Close() require.NoError(t, rpc.Dial(ctx)) ch := make(chan *evmtypes.Head) @@ -177,7 +184,7 @@ func TestRPCClient_SubscribeNewHead(t *testing.T) { } server := createRPCServer() - rpc := client.NewRPCClient(lggr, *server.URL, nil, "rpc", 1, chainId, commonclient.Primary, 0, tests.TestInterval, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + rpc := client.NewRPCClient(lggr, server.URL, nil, "rpc", 1, chainId, commonclient.Primary, 0, tests.TestInterval, commonclient.QueryTimeout, commonclient.QueryTimeout, "") defer rpc.Close() require.NoError(t, rpc.Dial(ctx)) latest, highestUserObservations := rpc.GetInterceptedChainInfo() @@ -201,7 +208,7 @@ func TestRPCClient_SubscribeNewHead(t *testing.T) { server := testutils.NewWSServer(t, chainId, serverCallBack) wsURL := server.WSURL() - rpc := client.NewRPCClient(lggr, *wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + rpc := client.NewRPCClient(lggr, wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") defer rpc.Close() require.NoError(t, rpc.Dial(ctx)) var wg sync.WaitGroup @@ -225,7 +232,7 @@ func TestRPCClient_SubscribeNewHead(t *testing.T) { t.Run("Block's chain ID matched configured", func(t *testing.T) { server := testutils.NewWSServer(t, chainId, serverCallBack) wsURL := server.WSURL() - rpc := client.NewRPCClient(lggr, *wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + rpc := client.NewRPCClient(lggr, wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") defer rpc.Close() require.NoError(t, rpc.Dial(ctx)) ch := make(chan *evmtypes.Head) @@ -242,7 +249,7 @@ func TestRPCClient_SubscribeNewHead(t *testing.T) { }) wsURL := server.WSURL() observedLggr, observed := logger.TestObserved(t, zap.DebugLevel) - rpc := client.NewRPCClient(observedLggr, *wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + rpc := client.NewRPCClient(observedLggr, wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") require.NoError(t, rpc.Dial(ctx)) server.Close() _, err := rpc.SubscribeNewHead(ctx, make(chan *evmtypes.Head)) @@ -253,7 +260,7 @@ func TestRPCClient_SubscribeNewHead(t *testing.T) { server := testutils.NewWSServer(t, chainId, serverCallBack) wsURL := server.WSURL() - rpc := client.NewRPCClient(lggr, *wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + rpc := client.NewRPCClient(lggr, wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") defer rpc.Close() require.NoError(t, rpc.Dial(ctx)) @@ -266,7 +273,7 @@ func TestRPCClient_SubscribeNewHead(t *testing.T) { server := testutils.NewWSServer(t, chainId, serverCallBack) wsURL := server.WSURL() - rpc := client.NewRPCClient(lggr, *wsURL, &url.URL{}, "rpc", 1, chainId, commonclient.Primary, 0, 1, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + rpc := client.NewRPCClient(lggr, wsURL, &url.URL{}, "rpc", 1, chainId, commonclient.Primary, 0, 1, commonclient.QueryTimeout, commonclient.QueryTimeout, "") defer rpc.Close() require.NoError(t, rpc.Dial(ctx)) @@ -279,7 +286,7 @@ func TestRPCClient_SubscribeNewHead(t *testing.T) { server := testutils.NewWSServer(t, chainId, serverCallBack) wsURL := server.WSURL() - rpc := client.NewRPCClient(lggr, *wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + rpc := client.NewRPCClient(lggr, wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") defer rpc.Close() require.NoError(t, rpc.Dial(ctx)) @@ -291,7 +298,7 @@ func TestRPCClient_SubscribeNewHead(t *testing.T) { server := testutils.NewWSServer(t, chainId, serverCallBack) wsURL := server.WSURL() - rpc := client.NewRPCClient(lggr, *wsURL, &url.URL{}, "rpc", 1, chainId, commonclient.Primary, 0, 1, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + rpc := client.NewRPCClient(lggr, wsURL, &url.URL{}, "rpc", 1, chainId, commonclient.Primary, 0, 1, commonclient.QueryTimeout, commonclient.QueryTimeout, "") defer rpc.Close() require.NoError(t, rpc.Dial(ctx)) @@ -303,7 +310,7 @@ func TestRPCClient_SubscribeNewHead(t *testing.T) { server := testutils.NewWSServer(t, chainId, serverCallBack) wsURL := server.WSURL() - rpc := client.NewRPCClient(lggr, *wsURL, &url.URL{}, "rpc", 1, chainId, commonclient.Primary, 1, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + rpc := client.NewRPCClient(lggr, wsURL, &url.URL{}, "rpc", 1, chainId, commonclient.Primary, 1, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") defer rpc.Close() require.NoError(t, rpc.Dial(ctx)) @@ -314,7 +321,7 @@ func TestRPCClient_SubscribeNewHead(t *testing.T) { t.Run("Subscription error is properly wrapper", func(t *testing.T) { server := testutils.NewWSServer(t, chainId, serverCallBack) wsURL := server.WSURL() - rpc := client.NewRPCClient(lggr, *wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + rpc := client.NewRPCClient(lggr, wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") defer rpc.Close() require.NoError(t, rpc.Dial(ctx)) sub, err := rpc.SubscribeNewHead(ctx, make(chan *evmtypes.Head)) @@ -336,13 +343,22 @@ func TestRPCClient_SubscribeFilterLogs(t *testing.T) { lggr := logger.Test(t) ctx, cancel := context.WithTimeout(tests.Context(t), tests.WaitTimeout(t)) defer cancel() + t.Run("Failed SubscribeFilterLogs when WSURL is empty", func(t *testing.T) { + // ws is optional when LogBroadcaster is disabled, however SubscribeFilterLogs will return error if ws is missing + observedLggr, _ := logger.TestObserved(t, zap.DebugLevel) + rpcClient := client.NewRPCClient(observedLggr, nil, &url.URL{}, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + require.Nil(t, rpcClient.Dial(ctx)) + + _, err := rpcClient.SubscribeFilterLogs(ctx, ethereum.FilterQuery{}, make(chan types.Log)) + require.Equal(t, errors.New("SubscribeFilterLogs is not allowed without ws url"), err) + }) t.Run("Failed SubscribeFilterLogs logs and returns proper error", func(t *testing.T) { server := testutils.NewWSServer(t, chainId, func(reqMethod string, reqParams gjson.Result) (resp testutils.JSONRPCResponse) { return resp }) wsURL := server.WSURL() observedLggr, observed := logger.TestObserved(t, zap.DebugLevel) - rpc := client.NewRPCClient(observedLggr, *wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + rpc := client.NewRPCClient(observedLggr, wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") require.NoError(t, rpc.Dial(ctx)) server.Close() _, err := rpc.SubscribeFilterLogs(ctx, ethereum.FilterQuery{}, make(chan types.Log)) @@ -359,7 +375,7 @@ func TestRPCClient_SubscribeFilterLogs(t *testing.T) { return resp }) wsURL := server.WSURL() - rpc := client.NewRPCClient(lggr, *wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + rpc := client.NewRPCClient(lggr, wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") defer rpc.Close() require.NoError(t, rpc.Dial(ctx)) sub, err := rpc.SubscribeFilterLogs(ctx, ethereum.FilterQuery{}, make(chan types.Log)) @@ -408,7 +424,7 @@ func TestRPCClient_LatestFinalizedBlock(t *testing.T) { } server := createRPCServer() - rpc := client.NewRPCClient(lggr, *server.URL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") + rpc := client.NewRPCClient(lggr, server.URL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, "") require.NoError(t, rpc.Dial(ctx)) defer rpc.Close() server.Head = &evmtypes.Head{Number: 128} @@ -518,7 +534,7 @@ func TestRpcClientLargePayloadTimeout(t *testing.T) { // use something unreasonably large for RPC timeout to ensure that we use largePayloadRPCTimeout const rpcTimeout = time.Hour const largePayloadRPCTimeout = tests.TestInterval - rpc := client.NewRPCClient(logger.Test(t), *rpcURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, largePayloadRPCTimeout, rpcTimeout, "") + rpc := client.NewRPCClient(logger.Test(t), rpcURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, largePayloadRPCTimeout, rpcTimeout, "") require.NoError(t, rpc.Dial(ctx)) defer rpc.Close() err := testCase.Fn(ctx, rpc) @@ -558,7 +574,7 @@ func TestAstarCustomFinality(t *testing.T) { const expectedFinalizedBlockNumber = int64(4) const expectedFinalizedBlockHash = "0x7441e97acf83f555e0deefef86db636bc8a37eb84747603412884e4df4d22804" - rpcClient := client.NewRPCClient(logger.Test(t), *wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, chaintype.ChainAstar) + rpcClient := client.NewRPCClient(logger.Test(t), wsURL, nil, "rpc", 1, chainId, commonclient.Primary, 0, 0, commonclient.QueryTimeout, commonclient.QueryTimeout, chaintype.ChainAstar) defer rpcClient.Close() err := rpcClient.Dial(tests.Context(t)) require.NoError(t, err) diff --git a/core/chains/evm/config/toml/config.go b/core/chains/evm/config/toml/config.go index fd4039f5ea..895c498aac 100644 --- a/core/chains/evm/config/toml/config.go +++ b/core/chains/evm/config/toml/config.go @@ -23,7 +23,9 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils/big" ) -var ErrNotFound = errors.New("not found") +var ( + ErrNotFound = errors.New("not found") +) type HasEVMConfigs interface { EVMConfigs() EVMConfigs @@ -311,16 +313,27 @@ func (c *EVMConfig) ValidateConfig() (err error) { err = multierr.Append(err, commonconfig.ErrMissing{Name: "Nodes", Msg: "must have at least one node"}) } else { var hasPrimary bool - for _, n := range c.Nodes { + var logBroadcasterEnabled bool + if c.LogBroadcasterEnabled != nil { + logBroadcasterEnabled = *c.LogBroadcasterEnabled + } + + for i, n := range c.Nodes { if n.SendOnly != nil && *n.SendOnly { continue } + hasPrimary = true - break + + // if the node is a primary node, then the WS URL is required when LogBroadcaster is enabled + if logBroadcasterEnabled && (n.WSURL == nil || n.WSURL.IsZero()) { + err = multierr.Append(err, commonconfig.ErrMissing{Name: "Nodes", Msg: fmt.Sprintf("%vth node (primary) must have a valid WSURL when LogBroadcaster is enabled", i)}) + } } + if !hasPrimary { err = multierr.Append(err, commonconfig.ErrMissing{Name: "Nodes", - Msg: "must have at least one primary node with WSURL"}) + Msg: "must have at least one primary node"}) } } @@ -971,19 +984,8 @@ func (n *Node) ValidateConfig() (err error) { err = multierr.Append(err, commonconfig.ErrEmpty{Name: "Name", Msg: "required for all nodes"}) } - var sendOnly bool - if n.SendOnly != nil { - sendOnly = *n.SendOnly - } - if n.WSURL == nil { - if !sendOnly { - err = multierr.Append(err, commonconfig.ErrMissing{Name: "WSURL", Msg: "required for primary nodes"}) - } - } else if n.WSURL.IsZero() { - if !sendOnly { - err = multierr.Append(err, commonconfig.ErrEmpty{Name: "WSURL", Msg: "required for primary nodes"}) - } - } else { + // relax the check here as WSURL can potentially be empty if LogBroadcaster is disabled (checked in EVMConfig Validation) + if n.WSURL != nil && !n.WSURL.IsZero() { switch n.WSURL.Scheme { case "ws", "wss": default: diff --git a/core/config/docs/chains-evm.toml b/core/config/docs/chains-evm.toml index fc53b0cc70..fe83fe64d5 100644 --- a/core/config/docs/chains-evm.toml +++ b/core/config/docs/chains-evm.toml @@ -469,7 +469,7 @@ ObservationGracePeriod = '1s' # Default [[EVM.Nodes]] # Name is a unique (per-chain) identifier for this node. Name = 'foo' # Example -# WSURL is the WS(S) endpoint for this node. Required for primary nodes. +# WSURL is the WS(S) endpoint for this node. Required for primary nodes when `LogBroadcasterEnabled` is `true` WSURL = 'wss://web.socket/test' # Example # HTTPURL is the HTTP(S) endpoint for this node. Required for all nodes. HTTPURL = 'https://foo.web' # Example diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index 97970df38f..0e5c3e095c 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -1341,7 +1341,8 @@ func TestConfig_Validate(t *testing.T) { - 1.ChainID: invalid value (1): duplicate - must be unique - 0.Nodes.1.Name: invalid value (foo): duplicate - must be unique - 3.Nodes.4.WSURL: invalid value (ws://dupe.com): duplicate - must be unique - - 0: 3 errors: + - 0: 4 errors: + - Nodes: missing: 0th node (primary) must have a valid WSURL when LogBroadcaster is enabled - GasEstimator.BumpTxDepth: invalid value (11): must be less than or equal to Transactions.MaxInFlight - GasEstimator: 6 errors: - BumpPercent: invalid value (1): may not be less than Geth's default of 10 @@ -1351,9 +1352,7 @@ func TestConfig_Validate(t *testing.T) { - PriceMax: invalid value (10 gwei): must be greater than or equal to PriceDefault - BlockHistory.BlockHistorySize: invalid value (0): must be greater than or equal to 1 with BlockHistory Mode - Nodes: 2 errors: - - 0: 2 errors: - - WSURL: missing: required for primary nodes - - HTTPURL: missing: required for all nodes + - 0.HTTPURL: missing: required for all nodes - 1.HTTPURL: missing: required for all nodes - 1: 10 errors: - ChainType: invalid value (Foo): must not be set with this chain id @@ -1374,18 +1373,19 @@ func TestConfig_Validate(t *testing.T) { - ChainType: invalid value (Arbitrum): must be one of arbitrum, astar, celo, gnosis, hedera, kroma, mantle, metis, optimismBedrock, scroll, wemix, xlayer, zkevm, zksync or omitted - FinalityDepth: invalid value (0): must be greater than or equal to 1 - MinIncomingConfirmations: invalid value (0): must be greater than or equal to 1 - - 3.Nodes: 5 errors: - - 0: 3 errors: + - 3: 3 errors: + - Nodes: missing: 0th node (primary) must have a valid WSURL when LogBroadcaster is enabled + - Nodes: missing: 2th node (primary) must have a valid WSURL when LogBroadcaster is enabled + - Nodes: 5 errors: + - 0: 2 errors: - Name: missing: required for all nodes - - WSURL: missing: required for primary nodes - HTTPURL: empty: required for all nodes - 1: 3 errors: - Name: missing: required for all nodes - WSURL: invalid value (http): must be ws or wss - HTTPURL: missing: required for all nodes - - 2: 3 errors: + - 2: 2 errors: - Name: empty: required for all nodes - - WSURL: missing: required for primary nodes - HTTPURL: invalid value (ws): must be http or https - 3.HTTPURL: missing: required for all nodes - 4.HTTPURL: missing: required for all nodes diff --git a/docs/CONFIG.md b/docs/CONFIG.md index c0bb07347c..3cea87d324 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -11338,7 +11338,7 @@ Name is a unique (per-chain) identifier for this node. ```toml WSURL = 'wss://web.socket/test' # Example ``` -WSURL is the WS(S) endpoint for this node. Required for primary nodes. +WSURL is the WS(S) endpoint for this node. Required for primary nodes when `LogBroadcasterEnabled` is `true` ### HTTPURL ```toml From c30820028f4f177e25868bd5a12cc7f2fc06c035 Mon Sep 17 00:00:00 2001 From: Joe Huang Date: Thu, 24 Oct 2024 14:38:49 -0500 Subject: [PATCH 03/10] Config validation requires ws url when http polling disabled (#14929) * add test * add more test * add changeset (cherry picked from commit da5910eda98882f8a1b9ccc52b99afa223fb3997) --- .changeset/four-kangaroos-appear.md | 5 +++++ core/chains/evm/config/toml/config.go | 17 ++++++++++++++--- core/services/chainlink/config_test.go | 3 ++- .../chainlink/testdata/config-invalid.toml | 19 +++++++++++++++++++ 4 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 .changeset/four-kangaroos-appear.md diff --git a/.changeset/four-kangaroos-appear.md b/.changeset/four-kangaroos-appear.md new file mode 100644 index 0000000000..b8ef32ff69 --- /dev/null +++ b/.changeset/four-kangaroos-appear.md @@ -0,0 +1,5 @@ +--- +"chainlink": patch +--- + +Add config validation so it requires ws url when http polling disabled #bugfix diff --git a/core/chains/evm/config/toml/config.go b/core/chains/evm/config/toml/config.go index 895c498aac..480d94d5d4 100644 --- a/core/chains/evm/config/toml/config.go +++ b/core/chains/evm/config/toml/config.go @@ -314,10 +314,15 @@ func (c *EVMConfig) ValidateConfig() (err error) { } else { var hasPrimary bool var logBroadcasterEnabled bool + var newHeadsPollingInterval commonconfig.Duration if c.LogBroadcasterEnabled != nil { logBroadcasterEnabled = *c.LogBroadcasterEnabled } + if c.NodePool.NewHeadsPollInterval != nil { + newHeadsPollingInterval = *c.NodePool.NewHeadsPollInterval + } + for i, n := range c.Nodes { if n.SendOnly != nil && *n.SendOnly { continue @@ -325,9 +330,15 @@ func (c *EVMConfig) ValidateConfig() (err error) { hasPrimary = true - // if the node is a primary node, then the WS URL is required when LogBroadcaster is enabled - if logBroadcasterEnabled && (n.WSURL == nil || n.WSURL.IsZero()) { - err = multierr.Append(err, commonconfig.ErrMissing{Name: "Nodes", Msg: fmt.Sprintf("%vth node (primary) must have a valid WSURL when LogBroadcaster is enabled", i)}) + // if the node is a primary node, then the WS URL is required when + // 1. LogBroadcaster is enabled + // 2. The http polling is disabled (newHeadsPollingInterval == 0) + if n.WSURL == nil || n.WSURL.IsZero() { + if logBroadcasterEnabled { + err = multierr.Append(err, commonconfig.ErrMissing{Name: "Nodes", Msg: fmt.Sprintf("%vth node (primary) must have a valid WSURL when LogBroadcaster is enabled", i)}) + } else if newHeadsPollingInterval.Duration() == 0 { + err = multierr.Append(err, commonconfig.ErrMissing{Name: "Nodes", Msg: fmt.Sprintf("%vth node (primary) must have a valid WSURL when http polling is disabled", i)}) + } } } diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index 0e5c3e095c..9caf37ccf2 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -1337,7 +1337,7 @@ func TestConfig_Validate(t *testing.T) { - LDAP.RunUserGroupCN: invalid value (): LDAP ReadUserGroupCN can not be empty - LDAP.RunUserGroupCN: invalid value (): LDAP RunUserGroupCN can not be empty - LDAP.ReadUserGroupCN: invalid value (): LDAP ReadUserGroupCN can not be empty - - EVM: 9 errors: + - EVM: 10 errors: - 1.ChainID: invalid value (1): duplicate - must be unique - 0.Nodes.1.Name: invalid value (foo): duplicate - must be unique - 3.Nodes.4.WSURL: invalid value (ws://dupe.com): duplicate - must be unique @@ -1393,6 +1393,7 @@ func TestConfig_Validate(t *testing.T) { - ChainID: missing: required for all chains - Nodes: missing: must have at least one node - 5.Transactions.AutoPurge.DetectionApiUrl: invalid value (): must be set for scroll + - 6.Nodes: missing: 0th node (primary) must have a valid WSURL when http polling is disabled - Cosmos: 5 errors: - 1.ChainID: invalid value (Malaga-420): duplicate - must be unique - 0.Nodes.1.Name: invalid value (test): duplicate - must be unique diff --git a/core/services/chainlink/testdata/config-invalid.toml b/core/services/chainlink/testdata/config-invalid.toml index ca22e68c22..411741b1b5 100644 --- a/core/services/chainlink/testdata/config-invalid.toml +++ b/core/services/chainlink/testdata/config-invalid.toml @@ -117,6 +117,25 @@ Name = 'scroll node' WSURL = 'ws://foo.bar' HTTPURl = 'http://foo.bar' +[[EVM]] +ChainID = '100' +LogBroadcasterEnabled = false + +[[EVM.Nodes]] +Name = 'failing-fake' +HTTPURl = 'http://foo.bar1' + +[[EVM]] +ChainID = '101' +LogBroadcasterEnabled = false + +[EVM.NodePool] +NewHeadsPollInterval = '1s' + +[[EVM.Nodes]] +Name = 'passing-fake' +HTTPURl = 'http://foo.bar2' + [[Cosmos]] ChainID = 'Malaga-420' From 98cc209464d04da7a60ed1addbf4e373aa2e1297 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Fri, 15 Nov 2024 18:32:22 +0100 Subject: [PATCH 04/10] attempt to fix upgrade CI --- .github/workflows/ccip-offchain-upgrade-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ccip-offchain-upgrade-tests.yml b/.github/workflows/ccip-offchain-upgrade-tests.yml index 7536488eb9..8ac8864748 100644 --- a/.github/workflows/ccip-offchain-upgrade-tests.yml +++ b/.github/workflows/ccip-offchain-upgrade-tests.yml @@ -141,6 +141,7 @@ jobs: with: # we just want to build the load tests suites: ccip-tests/load ccip-tests/smoke + repository: chainlink-ccip-tests QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} From 43bc470ae90d0d8fbd6207f19a36ac581976643a Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Fri, 15 Nov 2024 19:46:37 +0100 Subject: [PATCH 05/10] another attempt to fix ci --- .github/workflows/ccip-offchain-upgrade-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ccip-offchain-upgrade-tests.yml b/.github/workflows/ccip-offchain-upgrade-tests.yml index 8ac8864748..1c197777e1 100644 --- a/.github/workflows/ccip-offchain-upgrade-tests.yml +++ b/.github/workflows/ccip-offchain-upgrade-tests.yml @@ -142,6 +142,7 @@ jobs: # we just want to build the load tests suites: ccip-tests/load ccip-tests/smoke repository: chainlink-ccip-tests + tag: ${{ github.sha }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} From 4e26358214682824edb72ff6521d7f20b4b00ab5 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Sat, 16 Nov 2024 14:04:33 +0100 Subject: [PATCH 06/10] Revert "another attempt to fix ci" This reverts commit 43bc470ae90d0d8fbd6207f19a36ac581976643a. --- .github/workflows/ccip-offchain-upgrade-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ccip-offchain-upgrade-tests.yml b/.github/workflows/ccip-offchain-upgrade-tests.yml index 1c197777e1..8ac8864748 100644 --- a/.github/workflows/ccip-offchain-upgrade-tests.yml +++ b/.github/workflows/ccip-offchain-upgrade-tests.yml @@ -142,7 +142,6 @@ jobs: # we just want to build the load tests suites: ccip-tests/load ccip-tests/smoke repository: chainlink-ccip-tests - tag: ${{ github.sha }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} From 1c3fad2ad257932d728997a162f2f2358870bcdc Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Sat, 16 Nov 2024 14:05:18 +0100 Subject: [PATCH 07/10] Revert "attempt to fix upgrade CI" This reverts commit 98cc209464d04da7a60ed1addbf4e373aa2e1297. --- .github/workflows/ccip-offchain-upgrade-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ccip-offchain-upgrade-tests.yml b/.github/workflows/ccip-offchain-upgrade-tests.yml index 8ac8864748..7536488eb9 100644 --- a/.github/workflows/ccip-offchain-upgrade-tests.yml +++ b/.github/workflows/ccip-offchain-upgrade-tests.yml @@ -141,7 +141,6 @@ jobs: with: # we just want to build the load tests suites: ccip-tests/load ccip-tests/smoke - repository: chainlink-ccip-tests QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} From 7c96af712dbaccdbf5c020973f46cbf7bd4ea2a5 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Sat, 16 Nov 2024 08:06:12 -0500 Subject: [PATCH 08/10] Use explicit test image tag (#1541) ## Motivation ## Solution --- .github/workflows/ccip-offchain-upgrade-tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ccip-offchain-upgrade-tests.yml b/.github/workflows/ccip-offchain-upgrade-tests.yml index 7536488eb9..fa84a1c929 100644 --- a/.github/workflows/ccip-offchain-upgrade-tests.yml +++ b/.github/workflows/ccip-offchain-upgrade-tests.yml @@ -113,6 +113,8 @@ jobs: if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' needs: [ changes ] environment: integration + outputs: + tag: ${{ steps.build-test-image.outputs.test_image_tag }} permissions: id-token: write contents: read @@ -136,6 +138,7 @@ jobs: repository: smartcontractkit/ccip ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Build Test Image + id: build-test-image if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' uses: smartcontractkit/.github/actions/ctf-build-test-image@a5e4f4c8fbb8e15ab2ad131552eca6ac83c4f4b3 # ctf-build-test-image@0.1.0 with: @@ -249,7 +252,7 @@ jobs: env: BASE64_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} TEST_BASE64_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} - ENV_JOB_IMAGE: ${{ env.ENV_JOB_IMAGE_BASE }}:${{ github.sha }} + ENV_JOB_IMAGE: ${{ env.ENV_JOB_IMAGE_BASE }}:${{ needs.build-test-image.outputs.tag }} TEST_SUITE: smoke TEST_ARGS: -test.timeout 30m TEST_LOG_LEVEL: info From bcf938621031e02854e5e7818bf140516acb5693 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Sun, 17 Nov 2024 10:56:54 +0100 Subject: [PATCH 09/10] Revert "Revert "attempt to fix upgrade CI"" This reverts commit 1c3fad2ad257932d728997a162f2f2358870bcdc. --- .github/workflows/ccip-offchain-upgrade-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ccip-offchain-upgrade-tests.yml b/.github/workflows/ccip-offchain-upgrade-tests.yml index fa84a1c929..9ca4ffeea1 100644 --- a/.github/workflows/ccip-offchain-upgrade-tests.yml +++ b/.github/workflows/ccip-offchain-upgrade-tests.yml @@ -144,6 +144,7 @@ jobs: with: # we just want to build the load tests suites: ccip-tests/load ccip-tests/smoke + repository: chainlink-ccip-tests QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} From e2c7b033b0adc5704cb2b1193eabd1f53f1b1a72 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Sun, 17 Nov 2024 11:34:21 +0100 Subject: [PATCH 10/10] Revert "Revert "another attempt to fix ci"" This reverts commit 4e26358214682824edb72ff6521d7f20b4b00ab5. --- .github/workflows/ccip-offchain-upgrade-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ccip-offchain-upgrade-tests.yml b/.github/workflows/ccip-offchain-upgrade-tests.yml index 9ca4ffeea1..78569fc370 100644 --- a/.github/workflows/ccip-offchain-upgrade-tests.yml +++ b/.github/workflows/ccip-offchain-upgrade-tests.yml @@ -145,6 +145,7 @@ jobs: # we just want to build the load tests suites: ccip-tests/load ccip-tests/smoke repository: chainlink-ccip-tests + tag: ${{ github.sha }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}