diff --git a/integration-tests/actions/vrf/vrfv2/setup_steps.go b/integration-tests/actions/vrf/vrfv2/setup_steps.go index 5c441bf811e..c21ff0a11e6 100644 --- a/integration-tests/actions/vrf/vrfv2/setup_steps.go +++ b/integration-tests/actions/vrf/vrfv2/setup_steps.go @@ -320,7 +320,16 @@ func SetupVRFV2WrapperEnvironment( return wrapperContracts, &wrapperSubID, nil } -func SetupVRFV2Universe(ctx context.Context, t *testing.T, testConfig tc.TestConfig, chainID int64, cleanupFn func(), newEnvConfig vrfcommon.NewEnvConfig, l zerolog.Logger) (*test_env.CLClusterTestEnv, *vrfcommon.VRFContracts, *vrfcommon.VRFKeyData, map[vrfcommon.VRFNodeType]*vrfcommon.VRFNode, error) { +func SetupVRFV2Universe( + ctx context.Context, + t *testing.T, + testConfig tc.TestConfig, + chainID int64, + cleanupFn func(), + newEnvConfig vrfcommon.NewEnvConfig, + l zerolog.Logger, + chainlinkNodeLogScannerSettings test_env.ChainlinkNodeLogScannerSettings, +) (*test_env.CLClusterTestEnv, *vrfcommon.VRFContracts, *vrfcommon.VRFKeyData, map[vrfcommon.VRFNodeType]*vrfcommon.VRFNode, error) { var ( env *test_env.CLClusterTestEnv vrfContracts *vrfcommon.VRFContracts @@ -334,7 +343,7 @@ func SetupVRFV2Universe(ctx context.Context, t *testing.T, testConfig tc.TestCon return nil, nil, nil, nil, fmt.Errorf("%s, err: %w", "Error setting up VRF V2 for Existing env", err) } } else { - vrfContracts, vrfKey, env, nodeTypeToNodeMap, err = SetupVRFV2ForNewEnv(ctx, t, testConfig, chainID, cleanupFn, newEnvConfig, l) + vrfContracts, vrfKey, env, nodeTypeToNodeMap, err = SetupVRFV2ForNewEnv(ctx, t, testConfig, chainID, cleanupFn, newEnvConfig, l, chainlinkNodeLogScannerSettings) if err != nil { return nil, nil, nil, nil, fmt.Errorf("%s, err: %w", "Error setting up VRF V2 for New env", err) } @@ -350,17 +359,20 @@ func SetupVRFV2ForNewEnv( cleanupFn func(), newEnvConfig vrfcommon.NewEnvConfig, l zerolog.Logger, + chainlinkNodeLogScannerSettings test_env.ChainlinkNodeLogScannerSettings, ) (*vrfcommon.VRFContracts, *vrfcommon.VRFKeyData, *test_env.CLClusterTestEnv, map[vrfcommon.VRFNodeType]*vrfcommon.VRFNode, error) { network, err := actions.EthereumNetworkConfigFromConfig(l, &testConfig) if err != nil { return nil, nil, nil, nil, fmt.Errorf("%s, err: %w", "Error building ethereum network config", err) } + env, err := test_env.NewCLTestEnvBuilder(). WithTestInstance(t). WithTestConfig(&testConfig). WithPrivateEthereumNetwork(network.EthereumNetworkConfig). WithCLNodes(len(newEnvConfig.NodesToCreate)). WithFunding(big.NewFloat(*testConfig.Common.ChainlinkNodeFunding)). + WithChainlinkNodeLogScanner(chainlinkNodeLogScannerSettings). WithCustomCleanup(cleanupFn). Build() diff --git a/integration-tests/actions/vrf/vrfv2plus/setup_steps.go b/integration-tests/actions/vrf/vrfv2plus/setup_steps.go index ab973ffe110..fdb5592767a 100644 --- a/integration-tests/actions/vrf/vrfv2plus/setup_steps.go +++ b/integration-tests/actions/vrf/vrfv2plus/setup_steps.go @@ -371,7 +371,16 @@ func SetupVRFV2PlusWrapperEnvironment( return wrapperContracts, wrapperSubID, nil } -func SetupVRFV2PlusUniverse(ctx context.Context, t *testing.T, testConfig tc.TestConfig, chainID int64, cleanupFn func(), newEnvConfig vrfcommon.NewEnvConfig, l zerolog.Logger) (*test_env.CLClusterTestEnv, *vrfcommon.VRFContracts, *vrfcommon.VRFKeyData, map[vrfcommon.VRFNodeType]*vrfcommon.VRFNode, error) { +func SetupVRFV2PlusUniverse( + ctx context.Context, + t *testing.T, + testConfig tc.TestConfig, + chainID int64, + cleanupFn func(), + newEnvConfig vrfcommon.NewEnvConfig, + l zerolog.Logger, + chainlinkNodeLogScannerSettings test_env.ChainlinkNodeLogScannerSettings, +) (*test_env.CLClusterTestEnv, *vrfcommon.VRFContracts, *vrfcommon.VRFKeyData, map[vrfcommon.VRFNodeType]*vrfcommon.VRFNode, error) { var ( env *test_env.CLClusterTestEnv vrfContracts *vrfcommon.VRFContracts @@ -385,7 +394,7 @@ func SetupVRFV2PlusUniverse(ctx context.Context, t *testing.T, testConfig tc.Tes return nil, nil, nil, nil, fmt.Errorf("%s, err: %w", "Error setting up VRF V2 Plus for Existing env", err) } } else { - vrfContracts, vrfKey, env, nodeTypeToNode, err = SetupVRFV2PlusForNewEnv(ctx, t, testConfig, chainID, cleanupFn, newEnvConfig, l) + vrfContracts, vrfKey, env, nodeTypeToNode, err = SetupVRFV2PlusForNewEnv(ctx, t, testConfig, chainID, cleanupFn, newEnvConfig, l, chainlinkNodeLogScannerSettings) if err != nil { return nil, nil, nil, nil, fmt.Errorf("%s, err: %w", "Error setting up VRF V2 Plus for New env", err) } @@ -401,6 +410,7 @@ func SetupVRFV2PlusForNewEnv( cleanupFn func(), newEnvConfig vrfcommon.NewEnvConfig, l zerolog.Logger, + chainlinkNodeLogScannerSettings test_env.ChainlinkNodeLogScannerSettings, ) (*vrfcommon.VRFContracts, *vrfcommon.VRFKeyData, *test_env.CLClusterTestEnv, map[vrfcommon.VRFNodeType]*vrfcommon.VRFNode, error) { network, err := actions.EthereumNetworkConfigFromConfig(l, &testConfig) if err != nil { @@ -412,6 +422,7 @@ func SetupVRFV2PlusForNewEnv( WithPrivateEthereumNetwork(network.EthereumNetworkConfig). WithCLNodes(len(newEnvConfig.NodesToCreate)). WithFunding(big.NewFloat(*testConfig.Common.ChainlinkNodeFunding)). + WithChainlinkNodeLogScanner(chainlinkNodeLogScannerSettings). WithCustomCleanup(cleanupFn). Build() if err != nil { diff --git a/integration-tests/docker/test_env/test_env_builder.go b/integration-tests/docker/test_env/test_env_builder.go index 19617b0c5f7..0089ce0778a 100644 --- a/integration-tests/docker/test_env/test_env_builder.go +++ b/integration-tests/docker/test_env/test_env_builder.go @@ -1,15 +1,19 @@ package test_env import ( - "errors" + "context" "fmt" "math/big" "os" + "slices" "testing" + "github.com/pkg/errors" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/smartcontractkit/seth" + "go.uber.org/zap/zapcore" + "golang.org/x/sync/errgroup" "github.com/smartcontractkit/chainlink-testing-framework/blockchain" ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config" @@ -17,6 +21,7 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/logging" "github.com/smartcontractkit/chainlink-testing-framework/logstream" "github.com/smartcontractkit/chainlink-testing-framework/networks" + "github.com/smartcontractkit/chainlink-testing-framework/testreporters" "github.com/smartcontractkit/chainlink-testing-framework/utils/osutil" evmcfg "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/toml" @@ -35,38 +40,65 @@ const ( CleanUpTypeCustom CleanUpType = "custom" ) +type ChainlinkNodeLogScannerSettings struct { + FailingLogLevel zapcore.Level + Threshold uint + AllowedMessages []testreporters.AllowedLogMessage +} + type CLTestEnvBuilder struct { - hasLogStream bool - hasKillgrave bool - hasForwarders bool - hasSeth bool - hasEVMClient bool - clNodeConfig *chainlink.Config - secretsConfig string - clNodesCount int - clNodesOpts []func(*ClNode) - customNodeCsaKeys []string - defaultNodeCsaKeys []string - l zerolog.Logger - t *testing.T - te *CLClusterTestEnv - isNonEVM bool - cleanUpType CleanUpType - cleanUpCustomFn func() - chainOptionsFn []ChainOption - evmNetworkOption []EVMNetworkOption - privateEthereumNetworks []*ctf_config.EthereumNetworkConfig - testConfig ctf_config.GlobalTestConfig + hasLogStream bool + hasKillgrave bool + hasForwarders bool + hasSeth bool + hasEVMClient bool + clNodeConfig *chainlink.Config + secretsConfig string + clNodesCount int + clNodesOpts []func(*ClNode) + customNodeCsaKeys []string + defaultNodeCsaKeys []string + l zerolog.Logger + t *testing.T + te *CLClusterTestEnv + isNonEVM bool + cleanUpType CleanUpType + cleanUpCustomFn func() + chainOptionsFn []ChainOption + evmNetworkOption []EVMNetworkOption + privateEthereumNetworks []*ctf_config.EthereumNetworkConfig + testConfig ctf_config.GlobalTestConfig + chainlinkNodeLogScannerSettings *ChainlinkNodeLogScannerSettings /* funding */ ETHFunds *big.Float } +var DefaultAllowedMessages = []testreporters.AllowedLogMessage{ + testreporters.NewAllowedLogMessage("Failed to get LINK balance", "Happens only when we deploy LINK token for test purposes. Harmless.", zapcore.ErrorLevel, testreporters.WarnAboutAllowedMsgs_No), +} + +var DefaultChainlinkNodeLogScannerSettings = ChainlinkNodeLogScannerSettings{ + FailingLogLevel: zapcore.DPanicLevel, + Threshold: 1, // we want to fail on the first concerning log + AllowedMessages: DefaultAllowedMessages, +} + +func GetDefaultChainlinkNodeLogScannerSettingsWithExtraAllowedMessages(extraAllowedMessages ...testreporters.AllowedLogMessage) ChainlinkNodeLogScannerSettings { + allowedMessages := append(DefaultAllowedMessages, extraAllowedMessages...) + return ChainlinkNodeLogScannerSettings{ + FailingLogLevel: zapcore.DPanicLevel, + Threshold: 1, + AllowedMessages: allowedMessages, + } +} + func NewCLTestEnvBuilder() *CLTestEnvBuilder { return &CLTestEnvBuilder{ - l: log.Logger, - hasLogStream: true, - hasEVMClient: true, + l: log.Logger, + hasLogStream: true, + hasEVMClient: true, + chainlinkNodeLogScannerSettings: &DefaultChainlinkNodeLogScannerSettings, } } @@ -114,6 +146,16 @@ func (b *CLTestEnvBuilder) WithoutLogStream() *CLTestEnvBuilder { return b } +func (b *CLTestEnvBuilder) WithoutChainlinkNodeLogScanner() *CLTestEnvBuilder { + b.chainlinkNodeLogScannerSettings = &ChainlinkNodeLogScannerSettings{} + return b +} + +func (b *CLTestEnvBuilder) WithChainlinkNodeLogScanner(settings ChainlinkNodeLogScannerSettings) *CLTestEnvBuilder { + b.chainlinkNodeLogScannerSettings = &settings + return b +} + func (b *CLTestEnvBuilder) WithCLNodes(clNodesCount int) *CLTestEnvBuilder { b.clNodesCount = clNodesCount return b @@ -231,6 +273,12 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) { } if b.hasLogStream { + loggingConfig := b.testConfig.GetLoggingConfig() + // we need to enable logging to file if we want to scan logs + if b.chainlinkNodeLogScannerSettings != nil && !slices.Contains(loggingConfig.LogStream.LogTargets, string(logstream.File)) { + b.l.Debug().Msg("Enabling logging to file in order to support Chainlink node log scanning") + loggingConfig.LogStream.LogTargets = append(loggingConfig.LogStream.LogTargets, string(logstream.File)) + } b.te.LogStream, err = logstream.NewLogStream(b.te.t, b.testConfig.GetLoggingConfig()) if err != nil { return nil, err @@ -279,11 +327,55 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) { b.l.Info().Str("Absolute path", logPath).Msg("LogStream logs folder location") } + var scanClNodeLogs = func() { + //filter out non-cl logs + logLocation := b.te.LogStream.GetLogLocation() + logFiles, err := testreporters.FindAllLogFilesToScan(logLocation, "cl-node") + if err != nil { + b.l.Warn().Err(err).Msg("Error looking for Chainlink Node log files to scan") + } else { + // we ignore the context returned by errgroup here, since we have no way of interrupting ongoing scanning of logs + verifyLogsGroup, _ := errgroup.WithContext(context.Background()) + for _, f := range logFiles { + file := f + verifyLogsGroup.Go(func() error { + logErr := testreporters.VerifyLogFile(file, b.chainlinkNodeLogScannerSettings.FailingLogLevel, b.chainlinkNodeLogScannerSettings.Threshold, b.chainlinkNodeLogScannerSettings.AllowedMessages...) + if logErr != nil { + return errors.Wrapf(logErr, "Found a concerning log in %s", file.Name()) + } + return nil + }) + } + if err := verifyLogsGroup.Wait(); err != nil { + b.l.Error().Err(err).Msg("Found a concerning log. Failing test.") + b.t.Fatalf("Found a concerning log in Chainklink Node logs: %v", err) + } + } + b.l.Info().Msg("Finished scanning Chainlink Node logs for concerning errors") + } + if b.t.Failed() || *b.testConfig.GetLoggingConfig().TestLogCollect { // we can't do much if this fails, so we just log the error in logstream - _ = b.te.LogStream.FlushAndShutdown() + flushErr := b.te.LogStream.FlushAndShutdown() + if flushErr != nil { + b.l.Error().Err(flushErr).Msg("Error flushing and shutting down LogStream") + return + } b.te.LogStream.PrintLogTargetsLocations() b.te.LogStream.SaveLogLocationInTestSummary() + + // if test hasn't failed, but we have chainlinkNodeLogScannerSettings, we should check the logs + if !b.t.Failed() && b.chainlinkNodeLogScannerSettings != nil { + scanClNodeLogs() + } + } else if b.chainlinkNodeLogScannerSettings != nil { + flushErr := b.te.LogStream.FlushAndShutdown() + if flushErr != nil { + b.l.Error().Err(flushErr).Msg("Error flushing and shutting down LogStream") + return + } + + scanClNodeLogs() } }) } @@ -303,6 +395,10 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) { } } + if b.te.LogStream == nil && b.chainlinkNodeLogScannerSettings != nil { + log.Warn().Msg("Chainlink node log scanner settings provided, but LogStream is not enabled. Ignoring Chainlink node log scanner settings, as no logs will be available.") + } + // in this case we will use the builder only to start chains, not the cluster, because currently we support only 1 network config per cluster if len(b.privateEthereumNetworks) > 1 { b.te.rpcProviders = make(map[int64]*test_env.RpcProvider) diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 50296d09764..c831f3f181d 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -28,7 +28,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.3 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240514153505-0ddba5aa4d2c - github.com/smartcontractkit/chainlink-testing-framework v1.28.11 + github.com/smartcontractkit/chainlink-testing-framework v1.28.12 github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 04f7899c2d7..dfacb294fd9 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1524,8 +1524,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240510181707-46b1311a5a8 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240510181707-46b1311a5a83/go.mod h1:RdAtOeBUWq2zByw2kEbwPlXaPIb7YlaDOmnn+nVUBJI= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240508155030-1024f2b55c69 h1:ssh/w3oXWu+C6bE88GuFRC1+0Bx/4ihsbc80XMLrl2k= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240508155030-1024f2b55c69/go.mod h1:VsfjhvWgjxqWja4q+FlXEtX5lu8BSxn10xRo6gi948g= -github.com/smartcontractkit/chainlink-testing-framework v1.28.11 h1:6IcgmK/6UAfmcOYSiXudVlOa60MkbObTxA68hSJT4O4= -github.com/smartcontractkit/chainlink-testing-framework v1.28.11/go.mod h1:x1zDOz8zcLjEvs9fNA9y/DMguLam/2+CJdpxX0+rM8A= +github.com/smartcontractkit/chainlink-testing-framework v1.28.12 h1:15ssos9DvWekvj6JjmiPjTYsj/uw12HvTWlm1FHdYaA= +github.com/smartcontractkit/chainlink-testing-framework v1.28.12/go.mod h1:x1zDOz8zcLjEvs9fNA9y/DMguLam/2+CJdpxX0+rM8A= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 h1:fX/xmGm1GBsD1ZZnooNT+eWA0hiTAqFlHzOC5CY4dy8= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 8840e1264f9..782408dff3c 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -17,7 +17,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.3 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240514153505-0ddba5aa4d2c - github.com/smartcontractkit/chainlink-testing-framework v1.28.11 + github.com/smartcontractkit/chainlink-testing-framework v1.28.12 github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8 github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 05159b26ef5..00112420705 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1514,8 +1514,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240510181707-46b1311a5a8 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240510181707-46b1311a5a83/go.mod h1:RdAtOeBUWq2zByw2kEbwPlXaPIb7YlaDOmnn+nVUBJI= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240508155030-1024f2b55c69 h1:ssh/w3oXWu+C6bE88GuFRC1+0Bx/4ihsbc80XMLrl2k= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240508155030-1024f2b55c69/go.mod h1:VsfjhvWgjxqWja4q+FlXEtX5lu8BSxn10xRo6gi948g= -github.com/smartcontractkit/chainlink-testing-framework v1.28.11 h1:6IcgmK/6UAfmcOYSiXudVlOa60MkbObTxA68hSJT4O4= -github.com/smartcontractkit/chainlink-testing-framework v1.28.11/go.mod h1:x1zDOz8zcLjEvs9fNA9y/DMguLam/2+CJdpxX0+rM8A= +github.com/smartcontractkit/chainlink-testing-framework v1.28.12 h1:15ssos9DvWekvj6JjmiPjTYsj/uw12HvTWlm1FHdYaA= +github.com/smartcontractkit/chainlink-testing-framework v1.28.12/go.mod h1:x1zDOz8zcLjEvs9fNA9y/DMguLam/2+CJdpxX0+rM8A= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 h1:fX/xmGm1GBsD1ZZnooNT+eWA0hiTAqFlHzOC5CY4dy8= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8= diff --git a/integration-tests/load/vrfv2/vrfv2_test.go b/integration-tests/load/vrfv2/vrfv2_test.go index ca41dd40ea9..a62e4cdd552 100644 --- a/integration-tests/load/vrfv2/vrfv2_test.go +++ b/integration-tests/load/vrfv2/vrfv2_test.go @@ -99,7 +99,7 @@ func TestVRFV2Performance(t *testing.T) { UseTestCoordinator: true, } - testEnv, vrfContracts, vrfKey, _, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, testConfig, chainID, cleanupFn, newEnvConfig, l) + testEnv, vrfContracts, vrfKey, _, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, testConfig, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "error setting up VRFV2 universe") evmClient, err := testEnv.GetEVMClient(chainID) @@ -243,7 +243,7 @@ func TestVRFV2BHSPerformance(t *testing.T) { UseTestCoordinator: true, } - testEnv, vrfContracts, vrfKey, _, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, testConfig, chainID, cleanupFn, newEnvConfig, l) + testEnv, vrfContracts, vrfKey, _, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, testConfig, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "error setting up VRFV2 universe") evmClient, err := testEnv.GetEVMClient(chainID) diff --git a/integration-tests/load/vrfv2plus/vrfv2plus_test.go b/integration-tests/load/vrfv2plus/vrfv2plus_test.go index e24b2ff17d8..2496e2b692b 100644 --- a/integration-tests/load/vrfv2plus/vrfv2plus_test.go +++ b/integration-tests/load/vrfv2plus/vrfv2plus_test.go @@ -97,7 +97,7 @@ func TestVRFV2PlusPerformance(t *testing.T) { NumberOfTxKeysToCreate: *vrfv2PlusConfig.General.NumberOfSendingKeysToCreate, } - testEnv, vrfContracts, vrfKey, _, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, testConfig, chainID, cleanupFn, newEnvConfig, l) + testEnv, vrfContracts, vrfKey, _, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, testConfig, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "error setting up VRFV2Plus universe") evmClient, err := testEnv.GetEVMClient(chainID) @@ -240,7 +240,7 @@ func TestVRFV2PlusBHSPerformance(t *testing.T) { NumberOfTxKeysToCreate: *vrfv2PlusConfig.General.NumberOfSendingKeysToCreate, } - testEnv, vrfContracts, vrfKey, _, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, testConfig, chainID, cleanupFn, newEnvConfig, l) + testEnv, vrfContracts, vrfKey, _, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, testConfig, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "error setting up VRFV2Plus universe") evmClient, err := testEnv.GetEVMClient(chainID) diff --git a/integration-tests/smoke/log_poller_test.go b/integration-tests/smoke/log_poller_test.go index f75dfe96c7a..04327471e7f 100644 --- a/integration-tests/smoke/log_poller_test.go +++ b/integration-tests/smoke/log_poller_test.go @@ -11,9 +11,11 @@ import ( "github.com/onsi/gomega" "github.com/rs/zerolog" "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" "github.com/smartcontractkit/chainlink-testing-framework/logging" "github.com/smartcontractkit/chainlink-testing-framework/networks" + "github.com/smartcontractkit/chainlink-testing-framework/testreporters" "github.com/smartcontractkit/chainlink-testing-framework/utils/testcontext" actions_seth "github.com/smartcontractkit/chainlink/integration-tests/actions/seth" "github.com/smartcontractkit/chainlink/integration-tests/contracts" @@ -25,45 +27,52 @@ import ( core_logger "github.com/smartcontractkit/chainlink/v2/core/logger" ) +var logScannerSettings = test_env.GetDefaultChainlinkNodeLogScannerSettingsWithExtraAllowedMessages(testreporters.NewAllowedLogMessage( + "SLOW SQL QUERY", + "It is expected, because we are pausing the Postgres container", + zapcore.DPanicLevel, + testreporters.WarnAboutAllowedMsgs_No, +)) + // consistency test with no network disruptions with approximate emission of 1500-1600 logs per second for ~110-120 seconds // 6 filters are registered func TestLogPollerFewFiltersFixedDepth(t *testing.T) { - executeBasicLogPollerTest(t) + executeBasicLogPollerTest(t, test_env.DefaultChainlinkNodeLogScannerSettings) } func TestLogPollerFewFiltersFinalityTag(t *testing.T) { - executeBasicLogPollerTest(t) + executeBasicLogPollerTest(t, test_env.DefaultChainlinkNodeLogScannerSettings) } // consistency test with no network disruptions with approximate emission of 1000-1100 logs per second for ~110-120 seconds // 900 filters are registered func TestLogPollerManyFiltersFixedDepth(t *testing.T) { t.Skip("Execute manually, when needed as it runs for a long time") - executeBasicLogPollerTest(t) + executeBasicLogPollerTest(t, test_env.DefaultChainlinkNodeLogScannerSettings) } func TestLogPollerManyFiltersFinalityTag(t *testing.T) { t.Skip("Execute manually, when needed as it runs for a long time") - executeBasicLogPollerTest(t) + executeBasicLogPollerTest(t, test_env.DefaultChainlinkNodeLogScannerSettings) } -// consistency test that introduces random distruptions by pausing either Chainlink or Postgres containers for random interval of 5-20 seconds +// consistency test that introduces random disruptions by pausing either Chainlink or Postgres containers for random interval of 5-20 seconds // with approximate emission of 520-550 logs per second for ~110 seconds // 6 filters are registered func TestLogPollerWithChaosFixedDepth(t *testing.T) { - executeBasicLogPollerTest(t) + executeBasicLogPollerTest(t, logScannerSettings) } func TestLogPollerWithChaosFinalityTag(t *testing.T) { - executeBasicLogPollerTest(t) + executeBasicLogPollerTest(t, logScannerSettings) } func TestLogPollerWithChaosPostgresFixedDepth(t *testing.T) { - executeBasicLogPollerTest(t) + executeBasicLogPollerTest(t, logScannerSettings) } func TestLogPollerWithChaosPostgresFinalityTag(t *testing.T) { - executeBasicLogPollerTest(t) + executeBasicLogPollerTest(t, logScannerSettings) } // consistency test that registers filters after events were emitted and then triggers replay via API @@ -80,7 +89,7 @@ func TestLogPollerReplayFinalityTag(t *testing.T) { } // HELPER FUNCTIONS -func executeBasicLogPollerTest(t *testing.T) { +func executeBasicLogPollerTest(t *testing.T, logScannerSettings test_env.ChainlinkNodeLogScannerSettings) { testConfig, err := tc.GetConfig(t.Name(), tc.LogPoller) require.NoError(t, err, "Error getting config") overrideEphemeralAddressesCount(&testConfig) @@ -96,7 +105,7 @@ func executeBasicLogPollerTest(t *testing.T) { l := logging.GetTestLogger(t) coreLogger := core_logger.TestLogger(t) //needed by ORM ¯\_(ツ)_/¯ - lpTestEnv := prepareEnvironment(l, t, &testConfig) + lpTestEnv := prepareEnvironment(l, t, &testConfig, logScannerSettings) testEnv := lpTestEnv.testEnv ctx := testcontext.Get(t) @@ -177,7 +186,7 @@ func executeLogPollerReplay(t *testing.T, consistencyTimeout string) { l := logging.GetTestLogger(t) coreLogger := core_logger.TestLogger(t) //needed by ORM ¯\_(ツ)_/¯ - lpTestEnv := prepareEnvironment(l, t, &testConfig) + lpTestEnv := prepareEnvironment(l, t, &testConfig, test_env.DefaultChainlinkNodeLogScannerSettings) testEnv := lpTestEnv.testEnv ctx := testcontext.Get(t) @@ -262,7 +271,7 @@ type logPollerEnvironment struct { // prepareEnvironment prepares environment for log poller tests by starting DON, private Ethereum network, // deploying registry and log emitter contracts and registering log triggered upkeeps -func prepareEnvironment(l zerolog.Logger, t *testing.T, testConfig *tc.TestConfig) logPollerEnvironment { +func prepareEnvironment(l zerolog.Logger, t *testing.T, testConfig *tc.TestConfig, logScannerSettings test_env.ChainlinkNodeLogScannerSettings) logPollerEnvironment { cfg := testConfig.LogPoller if cfg.General.EventsToEmit == nil || len(cfg.General.EventsToEmit) == 0 { l.Warn().Msg("No events to emit specified, using all events from log emitter contract") @@ -287,6 +296,7 @@ func prepareEnvironment(l zerolog.Logger, t *testing.T, testConfig *tc.TestConfi *cfg.General.BackupLogPollerBlockDelay, *cfg.General.UseFinalityTag, testConfig, + logScannerSettings, ) _, upkeepIDs := actions_seth.DeployConsumers( diff --git a/integration-tests/smoke/vrfv2_test.go b/integration-tests/smoke/vrfv2_test.go index ce9f448ee66..946a73f0e18 100644 --- a/integration-tests/smoke/vrfv2_test.go +++ b/integration-tests/smoke/vrfv2_test.go @@ -14,10 +14,12 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" "github.com/onsi/gomega" "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" commonassets "github.com/smartcontractkit/chainlink-common/pkg/assets" "github.com/smartcontractkit/chainlink-testing-framework/logging" "github.com/smartcontractkit/chainlink-testing-framework/networks" + "github.com/smartcontractkit/chainlink-testing-framework/testreporters" "github.com/smartcontractkit/chainlink-testing-framework/utils/conversions" "github.com/smartcontractkit/chainlink-testing-framework/utils/ptr" "github.com/smartcontractkit/chainlink-testing-framework/utils/testcontext" @@ -75,7 +77,7 @@ func TestVRFv2Basic(t *testing.T) { UseTestCoordinator: false, } - testEnv, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + testEnv, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "Error setting up VRFV2 universe") evmClient, err := testEnv.GetEVMClient(chainID) require.NoError(t, err, "Getting EVM client shouldn't fail") @@ -610,7 +612,7 @@ func TestVRFv2MultipleSendingKeys(t *testing.T) { UseTestCoordinator: false, } - testEnv, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + testEnv, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "Error setting up VRFV2 universe") evmClient, err := testEnv.GetEVMClient(chainID) @@ -718,7 +720,7 @@ func TestVRFOwner(t *testing.T) { UseTestCoordinator: true, } - testEnv, vrfContracts, vrfKey, _, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + testEnv, vrfContracts, vrfKey, _, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "Error setting up VRFV2 universe") evmClient, err := testEnv.GetEVMClient(chainID) @@ -866,7 +868,7 @@ func TestVRFV2WithBHS(t *testing.T) { UseTestCoordinator: false, } - testEnv, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + testEnv, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "Error setting up VRFV2 universe") evmClient, err := testEnv.GetEVMClient(chainID) @@ -1070,7 +1072,20 @@ func TestVRFV2NodeReorg(t *testing.T) { UseTestCoordinator: false, } - env, vrfContracts, vrfKey, _, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + chainlinkNodeLogScannerSettings := test_env.GetDefaultChainlinkNodeLogScannerSettingsWithExtraAllowedMessages( + testreporters.NewAllowedLogMessage( + "This is a problem and either means a very deep re-org occurred", + "Test is expecting a reorg to occur", + zapcore.DPanicLevel, + testreporters.WarnAboutAllowedMsgs_No), + testreporters.NewAllowedLogMessage( + "Reorg greater than finality depth detected", + "Test is expecting a reorg to occur", + zapcore.DPanicLevel, + testreporters.WarnAboutAllowedMsgs_No), + ) + + env, vrfContracts, vrfKey, _, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, chainlinkNodeLogScannerSettings) require.NoError(t, err, "Error setting up VRFv2 universe") evmClient, err := env.GetEVMClient(chainID) @@ -1232,7 +1247,7 @@ func TestVRFv2BatchFulfillmentEnabledDisabled(t *testing.T) { UseTestCoordinator: false, } - env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "Error setting up VRFv2 universe") evmClient, err := env.GetEVMClient(chainID) diff --git a/integration-tests/smoke/vrfv2plus_test.go b/integration-tests/smoke/vrfv2plus_test.go index 0ef4e716733..6a64d08569a 100644 --- a/integration-tests/smoke/vrfv2plus_test.go +++ b/integration-tests/smoke/vrfv2plus_test.go @@ -13,10 +13,12 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" "github.com/onsi/gomega" "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" "github.com/smartcontractkit/chainlink-testing-framework/blockchain" "github.com/smartcontractkit/chainlink-testing-framework/logging" "github.com/smartcontractkit/chainlink-testing-framework/networks" + "github.com/smartcontractkit/chainlink-testing-framework/testreporters" "github.com/smartcontractkit/chainlink-testing-framework/utils/ptr" "github.com/smartcontractkit/chainlink-testing-framework/utils/testcontext" "github.com/smartcontractkit/chainlink/integration-tests/actions" @@ -76,7 +78,7 @@ func TestVRFv2Plus(t *testing.T) { UseTestCoordinator: false, } - env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "Error setting up VRFv2Plus universe") evmClient, err := env.GetEVMClient(chainID) @@ -775,7 +777,7 @@ func TestVRFv2PlusMultipleSendingKeys(t *testing.T) { UseTestCoordinator: false, } - env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "error setting up VRFV2Plus universe") evmClient, err := env.GetEVMClient(chainID) @@ -881,7 +883,7 @@ func TestVRFv2PlusMigration(t *testing.T) { UseTestCoordinator: false, } - env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "error setting up VRFV2Plus universe") evmClient, err := env.GetEVMClient(chainID) @@ -1309,7 +1311,7 @@ func TestVRFV2PlusWithBHS(t *testing.T) { UseTestCoordinator: false, } - env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "error setting up VRFV2Plus universe") evmClient, err := env.GetEVMClient(chainID) @@ -1532,8 +1534,14 @@ func TestVRFV2PlusWithBHF(t *testing.T) { UseTestCoordinator: false, } + chainlinkNodeLogScannerSettings := test_env.GetDefaultChainlinkNodeLogScannerSettingsWithExtraAllowedMessages(testreporters.NewAllowedLogMessage( + "Pipeline error", + "Test is expecting this error to occur", + zapcore.DPanicLevel, + testreporters.WarnAboutAllowedMsgs_No)) + env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse( - testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, chainlinkNodeLogScannerSettings) require.NoError(t, err) evmClient, err := env.GetEVMClient(chainID) require.NoError(t, err, "Getting EVM client shouldn't fail") @@ -1679,7 +1687,7 @@ func TestVRFv2PlusReplayAfterTimeout(t *testing.T) { config.VRFv2Plus.General.SubscriptionFundingAmountLink = ptr.Ptr(float64(0)) config.VRFv2Plus.General.SubscriptionFundingAmountNative = ptr.Ptr(float64(0)) - env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "error setting up VRFV2Plus universe") evmClient, err := env.GetEVMClient(chainID) @@ -1876,7 +1884,7 @@ func TestVRFv2PlusPendingBlockSimulationAndZeroConfirmationDelays(t *testing.T) config.VRFv2Plus.General.MinimumConfirmations = ptr.Ptr[uint16](0) config.VRFv2Plus.General.VRFJobSimulationBlock = ptr.Ptr[string]("pending") - env, vrfContracts, vrfKey, _, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + env, vrfContracts, vrfKey, _, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "error setting up VRFV2Plus universe") evmClient, err := env.GetEVMClient(chainID) @@ -1965,7 +1973,20 @@ func TestVRFv2PlusNodeReorg(t *testing.T) { UseTestCoordinator: false, } - env, vrfContracts, vrfKey, _, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + chainlinkNodeLogScannerSettings := test_env.GetDefaultChainlinkNodeLogScannerSettingsWithExtraAllowedMessages( + testreporters.NewAllowedLogMessage( + "This is a problem and either means a very deep re-org occurred", + "Test is expecting a reorg to occur", + zapcore.DPanicLevel, + testreporters.WarnAboutAllowedMsgs_No), + testreporters.NewAllowedLogMessage( + "Reorg greater than finality depth detected", + "Test is expecting a reorg to occur", + zapcore.DPanicLevel, + testreporters.WarnAboutAllowedMsgs_No), + ) + + env, vrfContracts, vrfKey, _, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, chainlinkNodeLogScannerSettings) require.NoError(t, err, "Error setting up VRFv2Plus universe") evmClient, err := env.GetEVMClient(chainID) @@ -2125,7 +2146,7 @@ func TestVRFv2PlusBatchFulfillmentEnabledDisabled(t *testing.T) { UseTestCoordinator: false, } - env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "Error setting up VRFv2Plus universe") evmClient, err := env.GetEVMClient(chainID) diff --git a/integration-tests/universal/log_poller/helpers.go b/integration-tests/universal/log_poller/helpers.go index 07334aa79ae..d732ad4af5d 100644 --- a/integration-tests/universal/log_poller/helpers.go +++ b/integration-tests/universal/log_poller/helpers.go @@ -1013,7 +1013,7 @@ func ExecuteChaosExperiment(l zerolog.Logger, testEnv *test_env.CLClusterTestEnv <-guardChan wg.Done() current := i + 1 - l.Info().Str("Current/Total", fmt.Sprintf("%d/%d", current, testConfig.LogPoller.ChaosConfig.ExperimentCount)).Msg("Done with experiment") + l.Info().Str("Current/Total", fmt.Sprintf("%d/%d", current, *testConfig.LogPoller.ChaosConfig.ExperimentCount)).Msg("Done with experiment") }() chaosChan <- chaosPauseSyncFn(l, sethClient, testEnv.ClCluster, *testConfig.LogPoller.ChaosConfig.TargetComponent) time.Sleep(10 * time.Second) @@ -1115,6 +1115,7 @@ func SetupLogPollerTestDocker( backupPollingInterval uint64, finalityTagEnabled bool, testConfig *tc.TestConfig, + logScannerSettings test_env.ChainlinkNodeLogScannerSettings, ) ( *seth.Client, []*client.ChainlinkClient, @@ -1179,6 +1180,7 @@ func SetupLogPollerTestDocker( WithFunding(big.NewFloat(chainlinkNodeFunding)). WithChainOptions(logPolllerSettingsFn). EVMNetworkOptions(evmNetworkSettingsFn). + WithChainlinkNodeLogScanner(logScannerSettings). WithStandardCleanup(). WithSeth(). Build() @@ -1344,6 +1346,7 @@ func FluentlyCheckIfAllNodesHaveLogCount(duration string, startBlock, endBlock i } l.Warn(). Msg("At least one CL node did not have expected log count. Retrying...") + time.Sleep(10 * time.Second) } return allNodesLogCountMatches, nil