diff --git a/.github/workflows/automation-benchmark-tests.yml b/.github/workflows/automation-benchmark-tests.yml index efe6d2eb59d..b2e5ad1638e 100644 --- a/.github/workflows/automation-benchmark-tests.yml +++ b/.github/workflows/automation-benchmark-tests.yml @@ -26,6 +26,7 @@ on: - SEPOLIA - BASE_GOERLI - ARBITRUM_SEPOLIA + - LINEA_GOERLI TestInputs: description: TestInputs required: false diff --git a/charts/chainlink-cluster/templates/chainlink-db-deployment.yaml b/charts/chainlink-cluster/templates/chainlink-db-deployment.yaml index f335130ea9f..91924ba5005 100644 --- a/charts/chainlink-cluster/templates/chainlink-db-deployment.yaml +++ b/charts/chainlink-cluster/templates/chainlink-db-deployment.yaml @@ -24,12 +24,20 @@ spec: selector: matchLabels: app: {{ $.Release.Name }}-db + # Used for testing. + # havoc-component-group and havoc-network-group are used by "havoc" chaos testing tool + havoc-component-group: db + havoc-network-group: db instance: {{ $cfg.name }}-db release: {{ $.Release.Name }} template: metadata: labels: app: {{ $.Release.Name }}-db + # Used for testing. + # havoc-component-group and havoc-network-group are used by "havoc" chaos testing tool + havoc-component-group: db + havoc-network-group: db instance: {{ $cfg.name }}-db release: {{ $.Release.Name }} {{- range $key, $value := $.Values.labels }} diff --git a/charts/chainlink-cluster/templates/chainlink-node-deployment.yaml b/charts/chainlink-cluster/templates/chainlink-node-deployment.yaml index 884cf0e535b..0ce16fd475b 100644 --- a/charts/chainlink-cluster/templates/chainlink-node-deployment.yaml +++ b/charts/chainlink-cluster/templates/chainlink-node-deployment.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ $.Release.Name }}-{{ $cfg.name }} + name: {{ if eq $index 0 }}{{ $.Release.Name }}-{{ $cfg.name }}-bootstrap{{ else }}{{ $.Release.Name }}-{{ $cfg.name }}{{ end }} spec: strategy: # Need to recreate the pod to deal with lease lock held by old pod. @@ -10,18 +10,31 @@ spec: selector: matchLabels: app: {{ $.Release.Name }} + # Used for testing. + # havoc-component-group and havoc-network-group are used by "havoc" chaos testing tool + {{ if eq $index 0 }}{{ else }} + havoc-component-group: node + {{ end }} + {{ if eq $index 0 }}{{ else }} + havoc-network-group: {{ if gt $index 2 }}"1"{{ else }}"2"{{ end }} + {{ end }} instance: {{ $cfg.name }} release: {{ $.Release.Name }} template: metadata: labels: app: {{ $.Release.Name }} + # Used for testing. + # havoc-component-group and havoc-network-group are used by "havoc" chaos testing tool + {{ if eq $index 0 }}{{ else }} + havoc-component-group: node + {{ end }} + {{ if eq $index 0 }}{{ else }} + havoc-network-group: {{ if gt $index 2 }}"1"{{ else }}"2"{{ end }} + {{ end }} + instance: {{ $cfg.name }} release: {{ $.Release.Name }} - # Used for testing. Role value should either be: bootstrap or node. - # There should only be one "bootstrap" node, the rest should be "node". - # Here we set the first node to be bootstrap, the rest to be node. - role: {{ if eq $index 0 }}bootstrap{{ else }}node{{ end }} {{- range $key, $value := $.Values.labels }} {{ $key }}: {{ $value | quote }} {{- end }} @@ -43,7 +56,7 @@ spec: {{- toYaml $.Values.chainlink.securityContext | nindent 12 }} image: {{ default "public.ecr.aws/chainlink/chainlink" $cfg.image }} imagePullPolicy: Always - command: ["bash", "-c", "while ! pg_isready -U postgres --host {{ $.Release.Name }}-db-{{ $cfg.name }} --port 5432; do echo \"waiting for database to start\"; sleep 1; done && chainlink -c /etc/node-secrets-volume/default.toml -c /etc/node-secrets-volume/overrides.toml -secrets /etc/node-secrets-volume/secrets.toml node start -d -p /etc/node-secrets-volume/node-password -a /etc/node-secrets-volume/apicredentials --vrfpassword=/etc/node-secrets-volume/apicredentials"] + command: [ "bash", "-c", "while ! pg_isready -U postgres --host {{ $.Release.Name }}-db-{{ $cfg.name }} --port 5432; do echo \"waiting for database to start\"; sleep 1; done && chainlink -c /etc/node-secrets-volume/default.toml -c /etc/node-secrets-volume/overrides.toml -secrets /etc/node-secrets-volume/secrets.toml node start -d -p /etc/node-secrets-volume/node-password -a /etc/node-secrets-volume/apicredentials --vrfpassword=/etc/node-secrets-volume/apicredentials" ] ports: - name: access containerPort: {{ $.Values.chainlink.web_port }} diff --git a/charts/chainlink-cluster/templates/geth-deployment.yaml b/charts/chainlink-cluster/templates/geth-deployment.yaml index 8d2d4d3c76c..abc7853d978 100644 --- a/charts/chainlink-cluster/templates/geth-deployment.yaml +++ b/charts/chainlink-cluster/templates/geth-deployment.yaml @@ -7,11 +7,19 @@ spec: selector: matchLabels: app: geth + # Used for testing. + # havoc-component-group and havoc-network-group are used by "havoc" chaos testing tool + havoc-component-group: "blockchain" + havoc-network-group: "blockchain" release: {{ .Release.Name }} template: metadata: labels: app: geth + # Used for testing. + # havoc-component-group and havoc-network-group are used by "havoc" chaos testing tool + havoc-component-group: "blockchain" + havoc-network-group: "blockchain" release: {{ .Release.Name }} annotations: {{- range $key, $value := .Values.podAnnotations }} diff --git a/core/chains/legacyevm/chain.go b/core/chains/legacyevm/chain.go index 0e0e1e65aca..92936299cdb 100644 --- a/core/chains/legacyevm/chain.go +++ b/core/chains/legacyevm/chain.go @@ -474,15 +474,13 @@ func newEthClientFromCfg(cfg evmconfig.NodePool, noNewHeadsThreshold time.Durati var sendonlys []commonclient.SendOnlyNode[*big.Int, evmclient.RPCCLient] for i, node := range nodes { if node.SendOnly != nil && *node.SendOnly { - name := fmt.Sprintf("eth-sendonly-rpc-%d", i) - rpc := evmclient.NewRPCClient(lggr, empty, (*url.URL)(node.HTTPURL), name, int32(i), chainID, + rpc := evmclient.NewRPCClient(lggr, empty, (*url.URL)(node.HTTPURL), *node.Name, int32(i), chainID, commonclient.Secondary) sendonly := commonclient.NewSendOnlyNode[*big.Int, evmclient.RPCCLient](lggr, (url.URL)(*node.HTTPURL), *node.Name, chainID, rpc) sendonlys = append(sendonlys, sendonly) } else { - name := fmt.Sprintf("eth-primary-rpc-%d", i) - rpc := evmclient.NewRPCClient(lggr, (url.URL)(*node.WSURL), (*url.URL)(node.HTTPURL), name, int32(i), + rpc := evmclient.NewRPCClient(lggr, (url.URL)(*node.WSURL), (*url.URL)(node.HTTPURL), *node.Name, int32(i), chainID, commonclient.Primary) primaryNode := commonclient.NewNode[*big.Int, *evmtypes.Head, evmclient.RPCCLient](cfg, noNewHeadsThreshold, lggr, (url.URL)(*node.WSURL), (*url.URL)(node.HTTPURL), *node.Name, int32(i), chainID, *node.Order, diff --git a/core/services/pipeline/internal/eautils/eautils.go b/core/services/pipeline/internal/eautils/eautils.go new file mode 100644 index 00000000000..30faa826b22 --- /dev/null +++ b/core/services/pipeline/internal/eautils/eautils.go @@ -0,0 +1,39 @@ +package eautils + +import ( + "encoding/json" + "net/http" +) + +type AdapterStatus struct { + ErrorMessage *string `json:"errorMessage"` + Error any `json:"error"` + StatusCode *int `json:"statusCode"` + ProviderStatusCode *int `json:"providerStatusCode"` +} + +func BestEffortExtractEAStatus(responseBytes []byte) (code int, ok bool) { + var status AdapterStatus + err := json.Unmarshal(responseBytes, &status) + if err != nil { + return 0, false + } + + if status.StatusCode == nil { + return 0, false + } + + if *status.StatusCode != http.StatusOK { + return *status.StatusCode, true + } + + if status.ProviderStatusCode != nil && *status.ProviderStatusCode != http.StatusOK { + return *status.ProviderStatusCode, true + } + + if status.Error != nil { + return http.StatusInternalServerError, true + } + + return *status.StatusCode, true +} diff --git a/core/services/pipeline/internal/eautils/eautils_test.go b/core/services/pipeline/internal/eautils/eautils_test.go new file mode 100644 index 00000000000..80183b80d2b --- /dev/null +++ b/core/services/pipeline/internal/eautils/eautils_test.go @@ -0,0 +1,61 @@ +package eautils + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBestEffortExtractEAStatus(t *testing.T) { + tests := []struct { + name string + arg []byte + expectCode int + expectOk bool + }{ + { + name: "invalid object", + arg: []byte(`{"error": "invalid json object" `), + expectCode: 0, + expectOk: false, + }, + { + name: "no status code in object", + arg: []byte(`{}`), + expectCode: 0, + expectOk: false, + }, + { + name: "invalid status code", + arg: []byte(`{"statusCode":400}`), + expectCode: http.StatusBadRequest, + expectOk: true, + }, + { + name: "invalid provider status code", + arg: []byte(`{"statusCode":200, "providerStatusCode":500}`), + expectCode: http.StatusInternalServerError, + expectOk: true, + }, + { + name: "valid statuses with error message", + arg: []byte(`{"statusCode":200, "providerStatusCode":200, "error": "unexpected error"}`), + expectCode: http.StatusInternalServerError, + expectOk: true, + }, + { + name: "valid status code", + arg: []byte(`{"statusCode":200}`), + expectCode: http.StatusOK, + expectOk: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + code, ok := BestEffortExtractEAStatus(tt.arg) + assert.Equal(t, tt.expectCode, code) + assert.Equal(t, tt.expectOk, ok) + }) + } +} diff --git a/core/services/pipeline/task.bridge.go b/core/services/pipeline/task.bridge.go index f9490ea791d..1da34d19134 100644 --- a/core/services/pipeline/task.bridge.go +++ b/core/services/pipeline/task.bridge.go @@ -16,6 +16,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/bridges" "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/pipeline/internal/eautils" ) // NOTE: These metrics generate a new label per bridge, this should be safe @@ -167,7 +168,13 @@ func (t *BridgeTask) Run(ctx context.Context, lggr logger.Logger, vars Vars, inp var cachedResponse bool responseBytes, statusCode, headers, elapsed, err := makeHTTPRequest(requestCtx, lggr, "POST", url, reqHeaders, requestData, t.httpClient, t.config.DefaultHTTPLimit()) - if err != nil { + + // check for external adapter response object status + if code, ok := eautils.BestEffortExtractEAStatus(responseBytes); ok { + statusCode = code + } + + if err != nil || statusCode != http.StatusOK { promBridgeErrors.WithLabelValues(t.Name).Inc() if cacheTTL == 0 { return Result{Error: err}, RunInfo{IsRetryable: isRetryableHTTPError(statusCode, err)} diff --git a/core/services/pipeline/task.bridge_test.go b/core/services/pipeline/task.bridge_test.go index bf1e63d6314..7673add1e35 100644 --- a/core/services/pipeline/task.bridge_test.go +++ b/core/services/pipeline/task.bridge_test.go @@ -19,7 +19,6 @@ import ( "github.com/shopspring/decimal" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gopkg.in/guregu/null.v4" commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config" "github.com/smartcontractkit/chainlink/v2/core/bridges" @@ -32,6 +31,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" "github.com/smartcontractkit/chainlink/v2/core/services/pg" "github.com/smartcontractkit/chainlink/v2/core/services/pipeline" + "github.com/smartcontractkit/chainlink/v2/core/services/pipeline/internal/eautils" "github.com/smartcontractkit/chainlink/v2/core/store/models" "github.com/smartcontractkit/chainlink/v2/core/utils" ) @@ -59,11 +59,43 @@ type adapterResponseData struct { // adapterResponse is the HTTP response as defined by the external adapter: // https://github.com/smartcontractkit/bnc-adapter type adapterResponse struct { - Data adapterResponseData `json:"data"` - ErrorMessage null.String `json:"errorMessage"` + eautils.AdapterStatus + Data adapterResponseData `json:"data"` } -func (pr adapterResponse) Result() *decimal.Decimal { +func (pr *adapterResponse) SetStatusCode(code int) { + pr.StatusCode = &code +} + +func (pr *adapterResponse) UnsetStatusCode() { + pr.StatusCode = nil +} + +func (pr *adapterResponse) SetProviderStatusCode(code int) { + pr.ProviderStatusCode = &code +} + +func (pr *adapterResponse) UnsetProviderStatusCode() { + pr.ProviderStatusCode = nil +} + +func (pr *adapterResponse) SetError(msg string) { + pr.Error = msg +} + +func (pr *adapterResponse) UnsetError() { + pr.Error = nil +} + +func (pr *adapterResponse) SetErrorMessage(msg string) { + pr.ErrorMessage = &msg +} + +func (pr *adapterResponse) UnsetErrorMessage() { + pr.ErrorMessage = nil +} + +func (pr *adapterResponse) Result() *decimal.Decimal { return pr.Data.Result } @@ -295,7 +327,7 @@ func TestBridgeTask_DoesNotReturnStaleResults(t *testing.T) { task.HelperSetDependencies(cfg.JobPipeline(), cfg.WebServer(), orm, specID, uuid.UUID{}, c) // Insert entry 1m in the past, stale value, should not be used in case of EA failure. - err = queryer.ExecQ(`INSERT INTO bridge_last_value(dot_id, spec_id, value, finished_at) + err = queryer.ExecQ(`INSERT INTO bridge_last_value(dot_id, spec_id, value, finished_at) VALUES($1, $2, $3, $4) ON CONFLICT ON CONSTRAINT bridge_last_value_pkey DO UPDATE SET value = $3, finished_at = $4;`, task.DotID(), specID, big.NewInt(9700).Bytes(), time.Now().Add(-1*time.Minute)) require.NoError(t, err) @@ -786,9 +818,10 @@ func TestBridgeTask_ErrorMessage(t *testing.T) { handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusTooManyRequests) - err := json.NewEncoder(w).Encode(adapterResponse{ - ErrorMessage: null.StringFrom("could not hit data fetcher"), - }) + + resp := &adapterResponse{} + resp.SetErrorMessage("could not hit data fetcher") + err := json.NewEncoder(w).Encode(resp) require.NoError(t, err) }) @@ -1016,3 +1049,93 @@ func TestBridgeTask_Headers(t *testing.T) { assert.Equal(t, []string{"Content-Length", "38", "Content-Type", "footype", "User-Agent", "Go-http-client/1.1", "X-Header-1", "foo", "X-Header-2", "bar"}, allHeaders(headers)) }) } + +func TestBridgeTask_AdapterResponseStatusFailure(t *testing.T) { + t.Parallel() + + db := pgtest.NewSqlxDB(t) + cfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) { + c.WebServer.BridgeCacheTTL = commonconfig.MustNewDuration(1 * time.Minute) + }) + + testAdapterResponse := &adapterResponse{ + Data: adapterResponseData{Result: &decimal.Zero}, + } + + queryer := pg.NewQ(db, logger.TestLogger(t), cfg.Database()) + s1 := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + err := json.NewEncoder(w).Encode(testAdapterResponse) + require.NoError(t, err) + })) + defer s1.Close() + + feedURL, err := url.ParseRequestURI(s1.URL) + require.NoError(t, err) + + orm := bridges.NewORM(db, logger.TestLogger(t), cfg.Database()) + _, bridge := cltest.MustCreateBridge(t, db, cltest.BridgeOpts{URL: feedURL.String()}, cfg.Database()) + + task := pipeline.BridgeTask{ + BaseTask: pipeline.NewBaseTask(0, "bridge", nil, nil, 0), + Name: bridge.Name.String(), + RequestData: btcUSDPairing, + } + c := clhttptest.NewTestLocalOnlyHTTPClient() + trORM := pipeline.NewORM(db, logger.TestLogger(t), cfg.Database(), cfg.JobPipeline().MaxSuccessfulRuns()) + specID, err := trORM.CreateSpec(pipeline.Pipeline{}, *models.NewInterval(5 * time.Minute), pg.WithParentCtx(testutils.Context(t))) + require.NoError(t, err) + task.HelperSetDependencies(cfg.JobPipeline(), cfg.WebServer(), orm, specID, uuid.UUID{}, c) + + // Insert entry 1m in the past, stale value, should not be used in case of EA failure. + err = queryer.ExecQ(`INSERT INTO bridge_last_value(dot_id, spec_id, value, finished_at) + VALUES($1, $2, $3, $4) ON CONFLICT ON CONSTRAINT bridge_last_value_pkey + DO UPDATE SET value = $3, finished_at = $4;`, task.DotID(), specID, big.NewInt(9700).Bytes(), time.Now()) + require.NoError(t, err) + + vars := pipeline.NewVarsFrom( + map[string]interface{}{ + "jobRun": map[string]interface{}{ + "meta": map[string]interface{}{ + "shouldFail": true, + }, + }, + }, + ) + + // expect all external adapter response status failures to be served from the cache + testAdapterResponse.SetStatusCode(http.StatusBadRequest) + result, runInfo := task.Run(testutils.Context(t), logger.TestLogger(t), vars, nil) + + require.NoError(t, result.Error) + require.NotNil(t, result.Value) + require.False(t, runInfo.IsRetryable) + require.False(t, runInfo.IsPending) + + testAdapterResponse.SetStatusCode(http.StatusOK) + testAdapterResponse.SetProviderStatusCode(http.StatusBadRequest) + result, runInfo = task.Run(testutils.Context(t), logger.TestLogger(t), vars, nil) + + require.NoError(t, result.Error) + require.NotNil(t, result.Value) + require.False(t, runInfo.IsRetryable) + require.False(t, runInfo.IsPending) + + testAdapterResponse.SetStatusCode(http.StatusOK) + testAdapterResponse.SetProviderStatusCode(http.StatusOK) + testAdapterResponse.SetError("some error") + result, runInfo = task.Run(testutils.Context(t), logger.TestLogger(t), vars, nil) + + require.NoError(t, result.Error) + require.NotNil(t, result.Value) + require.False(t, runInfo.IsRetryable) + require.False(t, runInfo.IsPending) + + testAdapterResponse.SetStatusCode(http.StatusInternalServerError) + result, runInfo = task.Run(testutils.Context(t), logger.TestLogger(t), vars, nil) + + require.NoError(t, result.Error) + require.NotNil(t, result.Value) + require.False(t, runInfo.IsRetryable) + require.False(t, runInfo.IsPending) +} diff --git a/core/services/pipeline/task.http_test.go b/core/services/pipeline/task.http_test.go index c0dd93df430..36ccc147a78 100644 --- a/core/services/pipeline/task.http_test.go +++ b/core/services/pipeline/task.http_test.go @@ -14,7 +14,6 @@ import ( "github.com/shopspring/decimal" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gopkg.in/guregu/null.v4" "github.com/smartcontractkit/chainlink/v2/core/bridges" "github.com/smartcontractkit/chainlink/v2/core/internal/cltest" @@ -264,9 +263,9 @@ func TestHTTPTask_ErrorMessage(t *testing.T) { handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusTooManyRequests) - err := json.NewEncoder(w).Encode(adapterResponse{ - ErrorMessage: null.StringFrom("could not hit data fetcher"), - }) + resp := &adapterResponse{} + resp.SetErrorMessage("could not hit data fetcher") + err := json.NewEncoder(w).Encode(resp) require.NoError(t, err) }) diff --git a/integration-tests/benchmark/keeper_test.go b/integration-tests/benchmark/keeper_test.go index 6d398c685e0..015dff1126c 100644 --- a/integration-tests/benchmark/keeper_test.go +++ b/integration-tests/benchmark/keeper_test.go @@ -295,6 +295,12 @@ var networkConfig = map[string]NetworkConfig{ deltaStage: 20 * time.Second, funding: big.NewFloat(ChainlinkNodeFunding), }, + "LineaGoerli": { + upkeepSLA: int64(120), + blockTime: time.Second, + deltaStage: 20 * time.Second, + funding: big.NewFloat(ChainlinkNodeFunding), + }, } func getEnv(key, fallback string) string { diff --git a/testdata/scripts/metrics/multi-node.txtar b/testdata/scripts/metrics/multi-node.txtar new file mode 100644 index 00000000000..c3928160443 --- /dev/null +++ b/testdata/scripts/metrics/multi-node.txtar @@ -0,0 +1,78 @@ +# Check that metrics specified in the expected_metrics are present in /metrics response +# start node +exec sh -c 'eval "echo \"$(cat config.toml.tmpl)\" > config.toml"' +exec chainlink node -c config.toml start -p password -a creds & + +# ensure node is up and running +env NODEURL=http://localhost:$PORT +exec curl --retry 10 --retry-max-time 60 --retry-connrefused $NODEURL + + +# Check +chmod 700 ./script.sh +exec sh -c './script.sh' + +-- script.sh -- + +maxRetries=5 +for retriesNum in $(seq 1 $maxRetries); do + passedAllChecks=true + curl $NODEURL/metrics > metrics.txt + while IFS= read -r expectedMetric; do + grep -q $expectedMetric metrics.txt && continue + + if [[ $retriesNum -ge $maxRetries ]]; then + cat metrics.txt + echo "FAIL Expected metric $expectedMetric to be present in GET /metrics response" + exit 1 + fi + + echo "Metric $expectedMetric is not present in GET /metrics response - retrying after 5s" + passedAllChecks=false + sleep 5 + break + done < expected_metrics.txt + + $passedAllChecks && break +done + +-- testdb.txt -- +CL_DATABASE_URL +-- testport.txt -- +PORT + +-- password -- +T.tLHkcmwePT/p,]sYuntjwHKAsrhm#4eRs4LuKHwvHejWYAC2JP4M8HimwgmbaZ +-- creds -- +notreal@fakeemail.ch +fj293fbBnlQ!f9vNs + +-- config.toml.tmpl -- +[Webserver] +HTTPPort = $PORT + +[[EVM]] +ChainID = '68472' + +[[EVM.Nodes]] +Name = 'BlueEVMPrimaryNode' +WSURL = 'wss://primaryfoo.bar/ws' +HTTPURL = 'https://primaryfoo.bar' + +[[EVM.Nodes]] +Name = 'YellowEVMPrimaryNode' +WSURL = 'wss://sendonlyfoo.bar/ws' +HTTPURL = 'https://sendonlyfoo.bar' +SendOnly = true + +-- expected_metrics.txt -- +evm_pool_rpc_node_dials_total{evmChainID="68472",nodeName="BlueEVMPrimaryNode"} +evm_pool_rpc_node_dials_total{evmChainID="68472",nodeName="YellowEVMPrimaryNode"} +multi_node_states{chainId="68472",network="EVM",state="Alive"} +multi_node_states{chainId="68472",network="EVM",state="Closed"} +multi_node_states{chainId="68472",network="EVM",state="Dialed"} +multi_node_states{chainId="68472",network="EVM",state="InvalidChainID"} +multi_node_states{chainId="68472",network="EVM",state="OutOfSync"} +multi_node_states{chainId="68472",network="EVM",state="Undialed"} +multi_node_states{chainId="68472",network="EVM",state="Unreachable"} +multi_node_states{chainId="68472",network="EVM",state="Unusable"} \ No newline at end of file