diff --git a/core/chains/evm/config/chain_scoped.go b/core/chains/evm/config/chain_scoped.go index c579da86c8c..69cc4c0a6ad 100644 --- a/core/chains/evm/config/chain_scoped.go +++ b/core/chains/evm/config/chain_scoped.go @@ -193,3 +193,7 @@ func (e *evmConfig) OperatorFactoryAddress() string { } return e.c.OperatorFactoryAddress.String() } + +func (e *evmConfig) LogPrunePageSize() uint32 { + return *e.c.LogPrunePageSize +} diff --git a/core/chains/evm/config/config.go b/core/chains/evm/config/config.go index 5b397ddd574..2cfc497f462 100644 --- a/core/chains/evm/config/config.go +++ b/core/chains/evm/config/config.go @@ -7,6 +7,7 @@ import ( gethcommon "github.com/ethereum/go-ethereum/common" commonassets "github.com/smartcontractkit/chainlink-common/pkg/assets" + commonconfig "github.com/smartcontractkit/chainlink/v2/common/config" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" "github.com/smartcontractkit/chainlink/v2/core/config" @@ -36,6 +37,7 @@ type EVM interface { LogBackfillBatchSize() uint32 LogKeepBlocksDepth() uint32 LogPollInterval() time.Duration + LogPrunePageSize() uint32 MinContractPayment() *commonassets.Link MinIncomingConfirmations() uint32 NonceAutoSync() bool diff --git a/core/chains/evm/config/toml/config.go b/core/chains/evm/config/toml/config.go index 6ebf3ed0a94..0ffe3549613 100644 --- a/core/chains/evm/config/toml/config.go +++ b/core/chains/evm/config/toml/config.go @@ -352,6 +352,7 @@ type Chain struct { LogBackfillBatchSize *uint32 LogPollInterval *commonconfig.Duration LogKeepBlocksDepth *uint32 + LogPrunePageSize *uint32 MinIncomingConfirmations *uint32 MinContractPayment *commonassets.Link NonceAutoSync *bool diff --git a/core/chains/evm/config/toml/defaults.go b/core/chains/evm/config/toml/defaults.go index adb91b3a1bc..242373fd4af 100644 --- a/core/chains/evm/config/toml/defaults.go +++ b/core/chains/evm/config/toml/defaults.go @@ -9,6 +9,7 @@ import ( "strings" cconfig "github.com/smartcontractkit/chainlink-common/pkg/config" + "github.com/smartcontractkit/chainlink/v2/common/config" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils/big" ) @@ -136,6 +137,9 @@ func (c *Chain) SetFrom(f *Chain) { if v := f.LogKeepBlocksDepth; v != nil { c.LogKeepBlocksDepth = v } + if v := f.LogPrunePageSize; v != nil { + c.LogPrunePageSize = v + } if v := f.MinIncomingConfirmations; v != nil { c.MinIncomingConfirmations = v } diff --git a/core/chains/evm/config/toml/defaults/fallback.toml b/core/chains/evm/config/toml/defaults/fallback.toml index 94fb83849bf..7b369142133 100644 --- a/core/chains/evm/config/toml/defaults/fallback.toml +++ b/core/chains/evm/config/toml/defaults/fallback.toml @@ -6,6 +6,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinContractPayment = '.00001 link' MinIncomingConfirmations = 3 NonceAutoSync = true diff --git a/core/chains/evm/forwarders/forwarder_manager_test.go b/core/chains/evm/forwarders/forwarder_manager_test.go index 5ef150aa5c3..4480e533525 100644 --- a/core/chains/evm/forwarders/forwarder_manager_test.go +++ b/core/chains/evm/forwarders/forwarder_manager_test.go @@ -60,7 +60,7 @@ func TestFwdMgr_MaybeForwardTransaction(t *testing.T) { t.Log(authorized) evmClient := client.NewSimulatedBackendClient(t, ec, testutils.FixtureChainID) - lp := logpoller.NewLogPoller(logpoller.NewORM(testutils.FixtureChainID, db, lggr, pgtest.NewQConfig(true)), evmClient, lggr, 100*time.Millisecond, false, 2, 3, 2, 1000) + lp := logpoller.NewLogPoller(logpoller.NewORM(testutils.FixtureChainID, db, lggr, pgtest.NewQConfig(true)), evmClient, lggr, 100*time.Millisecond, false, 2, 3, 2, 1000, 0) fwdMgr := forwarders.NewFwdMgr(db, evmClient, lp, lggr, evmcfg.EVM(), evmcfg.Database()) fwdMgr.ORM = forwarders.NewORM(db, logger.Test(t), cfg.Database()) @@ -113,7 +113,7 @@ func TestFwdMgr_AccountUnauthorizedToForward_SkipsForwarding(t *testing.T) { ec.Commit() evmClient := client.NewSimulatedBackendClient(t, ec, testutils.FixtureChainID) - lp := logpoller.NewLogPoller(logpoller.NewORM(testutils.FixtureChainID, db, lggr, pgtest.NewQConfig(true)), evmClient, lggr, 100*time.Millisecond, false, 2, 3, 2, 1000) + lp := logpoller.NewLogPoller(logpoller.NewORM(testutils.FixtureChainID, db, lggr, pgtest.NewQConfig(true)), evmClient, lggr, 100*time.Millisecond, false, 2, 3, 2, 1000, 0) fwdMgr := forwarders.NewFwdMgr(db, evmClient, lp, lggr, evmcfg.EVM(), evmcfg.Database()) fwdMgr.ORM = forwarders.NewORM(db, logger.Test(t), cfg.Database()) diff --git a/core/chains/evm/logpoller/helper_test.go b/core/chains/evm/logpoller/helper_test.go index 9e48690a249..cb0fbd247fc 100644 --- a/core/chains/evm/logpoller/helper_test.go +++ b/core/chains/evm/logpoller/helper_test.go @@ -20,6 +20,7 @@ import ( "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/logpoller" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/log_emitter" @@ -66,7 +67,7 @@ func SetupTH(t testing.TB, useFinalityTag bool, finalityDepth, backfillBatchSize // Mark genesis block as finalized to avoid any nulls in the tests head := esc.Backend().Blockchain().CurrentHeader() esc.Backend().Blockchain().SetFinalized(head) - lp := logpoller.NewLogPoller(o, esc, lggr, 1*time.Hour, useFinalityTag, finalityDepth, backfillBatchSize, rpcBatchSize, keepFinalizedBlocksDepth) + lp := logpoller.NewLogPoller(o, esc, lggr, 1*time.Hour, useFinalityTag, finalityDepth, backfillBatchSize, rpcBatchSize, keepFinalizedBlocksDepth, 0) emitterAddress1, _, emitter1, err := log_emitter.DeployLogEmitter(owner, ec) require.NoError(t, err) emitterAddress2, _, emitter2, err := log_emitter.DeployLogEmitter(owner, ec) diff --git a/core/chains/evm/logpoller/log_poller.go b/core/chains/evm/logpoller/log_poller.go index a2c35bec59f..ba617a2178b 100644 --- a/core/chains/evm/logpoller/log_poller.go +++ b/core/chains/evm/logpoller/log_poller.go @@ -76,7 +76,7 @@ type LogPollerTest interface { BackupPollAndSaveLogs(ctx context.Context, backupPollerBlockDelay int64) Filter(from, to *big.Int, bh *common.Hash) ethereum.FilterQuery GetReplayFromBlock(ctx context.Context, requested int64) (int64, error) - PruneOldBlocks(ctx context.Context) error + PruneOldBlocks(ctx context.Context) (bool, error) } type Client interface { @@ -106,6 +106,7 @@ type logPoller struct { backfillBatchSize int64 // batch size to use when backfilling finalized logs rpcBatchSize int64 // batch size to use for fallback RPC calls made in GetBlocks backupPollerNextBlock int64 + logPrunePageSize int64 filterMu sync.RWMutex filters map[string]Filter @@ -130,8 +131,7 @@ type logPoller struct { // // How fast that can be done depends largely on network speed and DB, but even for the fastest // support chain, polygon, which has 2s block times, we need RPCs roughly with <= 500ms latency -func NewLogPoller(orm ORM, ec Client, lggr logger.Logger, pollPeriod time.Duration, - useFinalityTag bool, finalityDepth int64, backfillBatchSize int64, rpcBatchSize int64, keepFinalizedBlocksDepth int64) *logPoller { +func NewLogPoller(orm ORM, ec Client, lggr logger.Logger, pollPeriod time.Duration, useFinalityTag bool, finalityDepth int64, backfillBatchSize int64, rpcBatchSize int64, keepFinalizedBlocksDepth int64, logsPrunePageSize int64) *logPoller { ctx, cancel := context.WithCancel(context.Background()) return &logPoller{ ctx: ctx, @@ -147,6 +147,7 @@ func NewLogPoller(orm ORM, ec Client, lggr logger.Logger, pollPeriod time.Durati backfillBatchSize: backfillBatchSize, rpcBatchSize: rpcBatchSize, keepFinalizedBlocksDepth: keepFinalizedBlocksDepth, + logPrunePageSize: logsPrunePageSize, filters: make(map[string]Filter), filterDirty: true, // Always build Filter on first call to cache an empty filter if nothing registered yet. } @@ -392,8 +393,9 @@ func (lp *logPoller) ReplayAsync(fromBlock int64) { func (lp *logPoller) Start(context.Context) error { return lp.StartOnce("LogPoller", func() error { - lp.wg.Add(1) + lp.wg.Add(2) go lp.run() + go lp.backgroundWorkerRun() return nil }) } @@ -439,8 +441,6 @@ func (lp *logPoller) run() { logPollTick := time.After(0) // stagger these somewhat, so they don't all run back-to-back backupLogPollTick := time.After(100 * time.Millisecond) - blockPruneTick := time.After(3 * time.Second) - logPruneTick := time.After(5 * time.Second) filtersLoaded := false loadFilters := func() error { @@ -545,15 +545,38 @@ func (lp *logPoller) run() { continue } lp.BackupPollAndSaveLogs(lp.ctx, backupPollerBlockDelay) + } + } +} + +func (lp *logPoller) backgroundWorkerRun() { + defer lp.wg.Done() + + // Avoid putting too much pressure on the database by staggering the pruning of old blocks and logs. + // Usually, node after restart will have some work to boot the plugins and other services. + // Deferring first prune by minutes reduces risk of putting too much pressure on the database. + blockPruneTick := time.After(5 * time.Minute) + logPruneTick := time.After(10 * time.Minute) + + for { + select { + case <-lp.ctx.Done(): + return case <-blockPruneTick: blockPruneTick = time.After(utils.WithJitter(lp.pollPeriod * 1000)) - if err := lp.PruneOldBlocks(lp.ctx); err != nil { + if allRemoved, err := lp.PruneOldBlocks(lp.ctx); err != nil { lp.lggr.Errorw("Unable to prune old blocks", "err", err) + } else if !allRemoved { + // Tick faster when cleanup can't keep up with the pace of new blocks + blockPruneTick = time.After(utils.WithJitter(lp.pollPeriod * 100)) } case <-logPruneTick: logPruneTick = time.After(utils.WithJitter(lp.pollPeriod * 2401)) // = 7^5 avoids common factors with 1000 - if err := lp.orm.DeleteExpiredLogs(pg.WithParentCtx(lp.ctx)); err != nil { - lp.lggr.Error(err) + if allRemoved, err := lp.PruneExpiredLogs(lp.ctx); err != nil { + lp.lggr.Errorw("Unable to prune expired logs", "err", err) + } else if !allRemoved { + // Tick faster when cleanup can't keep up with the pace of new logs + logPruneTick = time.After(utils.WithJitter(lp.pollPeriod * 241)) } } } @@ -933,22 +956,35 @@ func (lp *logPoller) findBlockAfterLCA(ctx context.Context, current *evmtypes.He } // PruneOldBlocks removes blocks that are > lp.keepFinalizedBlocksDepth behind the latest finalized block. -func (lp *logPoller) PruneOldBlocks(ctx context.Context) error { +// Returns whether all blocks eligible for pruning were removed. If logPrunePageSize is set to 0, it will always return true. +func (lp *logPoller) PruneOldBlocks(ctx context.Context) (bool, error) { latestBlock, err := lp.orm.SelectLatestBlock(pg.WithParentCtx(ctx)) if err != nil { - return err + return false, err } if latestBlock == nil { // No blocks saved yet. - return nil + return true, nil } if latestBlock.FinalizedBlockNumber <= lp.keepFinalizedBlocksDepth { // No-op, keep all blocks - return nil + return true, nil } // 1-2-3-4-5(finalized)-6-7(latest), keepFinalizedBlocksDepth=3 // Remove <= 2 - return lp.orm.DeleteBlocksBefore(latestBlock.FinalizedBlockNumber-lp.keepFinalizedBlocksDepth, pg.WithParentCtx(ctx)) + rowsRemoved, err := lp.orm.DeleteBlocksBefore( + latestBlock.FinalizedBlockNumber-lp.keepFinalizedBlocksDepth, + lp.logPrunePageSize, + pg.WithParentCtx(ctx), + ) + return lp.logPrunePageSize == 0 || rowsRemoved < lp.logPrunePageSize, err +} + +// PruneExpiredLogs logs that are older than their retention period defined in Filter. +// Returns whether all logs eligible for pruning were removed. If logPrunePageSize is set to 0, it will always return true. +func (lp *logPoller) PruneExpiredLogs(ctx context.Context) (bool, error) { + rowsRemoved, err := lp.orm.DeleteExpiredLogs(lp.logPrunePageSize, pg.WithParentCtx(ctx)) + return lp.logPrunePageSize == 0 || rowsRemoved < lp.logPrunePageSize, err } // Logs returns logs matching topics and address (exactly) in the given block range, diff --git a/core/chains/evm/logpoller/log_poller_internal_test.go b/core/chains/evm/logpoller/log_poller_internal_test.go index 899efebe42c..124c16d26f3 100644 --- a/core/chains/evm/logpoller/log_poller_internal_test.go +++ b/core/chains/evm/logpoller/log_poller_internal_test.go @@ -64,7 +64,7 @@ func TestLogPoller_RegisterFilter(t *testing.T) { orm := NewORM(chainID, db, lggr, pgtest.NewQConfig(true)) // Set up a test chain with a log emitting contract deployed. - lp := NewLogPoller(orm, nil, lggr, time.Hour, false, 1, 1, 2, 1000) + lp := NewLogPoller(orm, nil, lggr, time.Hour, false, 1, 1, 2, 1000, 0) // We expect a zero Filter if nothing registered yet. f := lp.Filter(nil, nil, nil) @@ -218,7 +218,7 @@ func TestLogPoller_BackupPollerStartup(t *testing.T) { ctx := testutils.Context(t) - lp := NewLogPoller(orm, ec, lggr, 1*time.Hour, false, 2, 3, 2, 1000) + lp := NewLogPoller(orm, ec, lggr, 1*time.Hour, false, 2, 3, 2, 1000, 0) lp.BackupPollAndSaveLogs(ctx, 100) assert.Equal(t, int64(0), lp.backupPollerNextBlock) assert.Equal(t, 1, observedLogs.FilterMessageSnippet("ran before first successful log poller run").Len()) @@ -258,7 +258,7 @@ func TestLogPoller_Replay(t *testing.T) { ec.On("HeadByNumber", mock.Anything, mock.Anything).Return(&head, nil) ec.On("FilterLogs", mock.Anything, mock.Anything).Return([]types.Log{log1}, nil).Once() ec.On("ConfiguredChainID").Return(chainID, nil) - lp := NewLogPoller(orm, ec, lggr, time.Hour, false, 3, 3, 3, 20) + lp := NewLogPoller(orm, ec, lggr, time.Hour, false, 3, 3, 3, 20, 0) // process 1 log in block 3 lp.PollAndSaveLogs(testutils.Context(t), 4) @@ -446,7 +446,7 @@ func Test_latestBlockAndFinalityDepth(t *testing.T) { ec := evmclimocks.NewClient(t) ec.On("HeadByNumber", mock.Anything, mock.Anything).Return(&head, nil) - lp := NewLogPoller(orm, ec, lggr, time.Hour, false, finalityDepth, 3, 3, 20) + lp := NewLogPoller(orm, ec, lggr, time.Hour, false, finalityDepth, 3, 3, 20, 0) latestBlock, lastFinalizedBlockNumber, err := lp.latestBlocks(testutils.Context(t)) require.NoError(t, err) require.Equal(t, latestBlock.Number, head.Number) @@ -470,7 +470,7 @@ func Test_latestBlockAndFinalityDepth(t *testing.T) { *(elems[1].Result.(*evmtypes.Head)) = evmtypes.Head{Number: expectedLastFinalizedBlockNumber, Hash: utils.RandomBytes32()} }) - lp := NewLogPoller(orm, ec, lggr, time.Hour, true, 3, 3, 3, 20) + lp := NewLogPoller(orm, ec, lggr, time.Hour, true, 3, 3, 3, 20, 0) latestBlock, lastFinalizedBlockNumber, err := lp.latestBlocks(testutils.Context(t)) require.NoError(t, err) @@ -488,7 +488,7 @@ func Test_latestBlockAndFinalityDepth(t *testing.T) { elems[1].Error = fmt.Errorf("some error") }) - lp := NewLogPoller(orm, ec, lggr, time.Hour, true, 3, 3, 3, 20) + lp := NewLogPoller(orm, ec, lggr, time.Hour, true, 3, 3, 3, 20, 0) _, _, err := lp.latestBlocks(testutils.Context(t)) require.Error(t, err) }) @@ -497,7 +497,7 @@ func Test_latestBlockAndFinalityDepth(t *testing.T) { ec := evmclimocks.NewClient(t) ec.On("BatchCallContext", mock.Anything, mock.Anything).Return(fmt.Errorf("some error")) - lp := NewLogPoller(orm, ec, lggr, time.Hour, true, 3, 3, 3, 20) + lp := NewLogPoller(orm, ec, lggr, time.Hour, true, 3, 3, 3, 20, 0) _, _, err := lp.latestBlocks(testutils.Context(t)) require.Error(t, err) }) @@ -506,7 +506,7 @@ func Test_latestBlockAndFinalityDepth(t *testing.T) { func benchmarkFilter(b *testing.B, nFilters, nAddresses, nEvents int) { lggr := logger.Test(b) - lp := NewLogPoller(nil, nil, lggr, 1*time.Hour, false, 2, 3, 2, 1000) + lp := NewLogPoller(nil, nil, lggr, 1*time.Hour, false, 2, 3, 2, 1000, 0) for i := 0; i < nFilters; i++ { var addresses []common.Address var events []common.Hash diff --git a/core/chains/evm/logpoller/log_poller_test.go b/core/chains/evm/logpoller/log_poller_test.go index ab02e783a28..5b894b8a19a 100644 --- a/core/chains/evm/logpoller/log_poller_test.go +++ b/core/chains/evm/logpoller/log_poller_test.go @@ -676,7 +676,7 @@ func TestLogPoller_SynchronizedWithGeth(t *testing.T) { }, 10e6) _, _, emitter1, err := log_emitter.DeployLogEmitter(owner, ec) require.NoError(t, err) - lp := logpoller.NewLogPoller(orm, client.NewSimulatedBackendClient(t, ec, chainID), lggr, 15*time.Second, false, int64(finalityDepth), 3, 2, 1000) + lp := logpoller.NewLogPoller(orm, client.NewSimulatedBackendClient(t, ec, chainID), lggr, 15*time.Second, false, int64(finalityDepth), 3, 2, 1000, 0) for i := 0; i < finalityDepth; i++ { // Have enough blocks that we could reorg the full finalityDepth-1. ec.Commit() } @@ -1306,7 +1306,7 @@ func TestLogPoller_DBErrorHandling(t *testing.T) { ec.Commit() ec.Commit() - lp := logpoller.NewLogPoller(o, client.NewSimulatedBackendClient(t, ec, chainID2), lggr, 1*time.Hour, false, 2, 3, 2, 1000) + lp := logpoller.NewLogPoller(o, client.NewSimulatedBackendClient(t, ec, chainID2), lggr, 1*time.Hour, false, 2, 3, 2, 1000, 0) err = lp.Replay(ctx, 5) // block number too high require.ErrorContains(t, err, "Invalid replay block number") @@ -1354,7 +1354,7 @@ func TestTooManyLogResults(t *testing.T) { chainID := testutils.NewRandomEVMChainID() db := pgtest.NewSqlxDB(t) o := logpoller.NewORM(chainID, db, lggr, pgtest.NewQConfig(true)) - lp := logpoller.NewLogPoller(o, ec, lggr, 1*time.Hour, false, 2, 20, 10, 1000) + lp := logpoller.NewLogPoller(o, ec, lggr, 1*time.Hour, false, 2, 20, 10, 1000, 0) expected := []int64{10, 5, 2, 1} clientErr := client.JsonError{ @@ -1671,11 +1671,14 @@ func Test_PruneOldBlocks(t *testing.T) { } if tt.wantErr { - require.Error(t, th.LogPoller.PruneOldBlocks(ctx)) + _, err := th.LogPoller.PruneOldBlocks(ctx) + require.Error(t, err) return } - require.NoError(t, th.LogPoller.PruneOldBlocks(ctx)) + allDeleted, err := th.LogPoller.PruneOldBlocks(ctx) + require.NoError(t, err) + assert.True(t, allDeleted) blocks, err := th.ORM.GetBlocksRange(0, math.MaxInt64, pg.WithParentCtx(ctx)) require.NoError(t, err) assert.Len(t, blocks, tt.blocksLeft) diff --git a/core/chains/evm/logpoller/observability.go b/core/chains/evm/logpoller/observability.go index a7a0d3c03d5..abb3246585b 100644 --- a/core/chains/evm/logpoller/observability.go +++ b/core/chains/evm/logpoller/observability.go @@ -122,9 +122,9 @@ func (o *ObservedORM) DeleteFilter(name string, qopts ...pg.QOpt) error { }) } -func (o *ObservedORM) DeleteBlocksBefore(end int64, qopts ...pg.QOpt) error { - return withObservedExec(o, "DeleteBlocksBefore", del, func() error { - return o.ORM.DeleteBlocksBefore(end, qopts...) +func (o *ObservedORM) DeleteBlocksBefore(end int64, limit int64, qopts ...pg.QOpt) (int64, error) { + return withObservedExecAndRowsAffected(o, "DeleteBlocksBefore", del, func() (int64, error) { + return o.ORM.DeleteBlocksBefore(end, limit, qopts...) }) } @@ -134,9 +134,9 @@ func (o *ObservedORM) DeleteLogsAndBlocksAfter(start int64, qopts ...pg.QOpt) er }) } -func (o *ObservedORM) DeleteExpiredLogs(qopts ...pg.QOpt) error { - return withObservedExec(o, "DeleteExpiredLogs", del, func() error { - return o.ORM.DeleteExpiredLogs(qopts...) +func (o *ObservedORM) DeleteExpiredLogs(limit int64, qopts ...pg.QOpt) (int64, error) { + return withObservedExecAndRowsAffected(o, "DeleteExpiredLogs", del, func() (int64, error) { + return o.ORM.DeleteExpiredLogs(limit, qopts...) }) } @@ -264,6 +264,22 @@ func withObservedQueryAndResults[T any](o *ObservedORM, queryName string, query return results, err } +func withObservedExecAndRowsAffected(o *ObservedORM, queryName string, queryType queryType, exec func() (int64, error)) (int64, error) { + queryStarted := time.Now() + rowsAffected, err := exec() + o.queryDuration. + WithLabelValues(o.chainId, queryName, string(queryType)). + Observe(float64(time.Since(queryStarted))) + + if err != nil { + o.datasetSize. + WithLabelValues(o.chainId, queryName, string(queryType)). + Set(float64(rowsAffected)) + } + + return rowsAffected, err +} + func withObservedQuery[T any](o *ObservedORM, queryName string, query func() (T, error)) (T, error) { queryStarted := time.Now() defer func() { diff --git a/core/chains/evm/logpoller/orm.go b/core/chains/evm/logpoller/orm.go index fe814e8a43c..c0e870870e6 100644 --- a/core/chains/evm/logpoller/orm.go +++ b/core/chains/evm/logpoller/orm.go @@ -30,9 +30,9 @@ type ORM interface { LoadFilters(qopts ...pg.QOpt) (map[string]Filter, error) DeleteFilter(name string, qopts ...pg.QOpt) error - DeleteBlocksBefore(end int64, qopts ...pg.QOpt) error + DeleteBlocksBefore(end int64, limit int64, qopts ...pg.QOpt) (int64, error) DeleteLogsAndBlocksAfter(start int64, qopts ...pg.QOpt) error - DeleteExpiredLogs(qopts ...pg.QOpt) error + DeleteExpiredLogs(limit int64, qopts ...pg.QOpt) (int64, error) GetBlocksRange(start int64, end int64, qopts ...pg.QOpt) ([]LogPollerBlock, error) SelectBlockByNumber(blockNumber int64, qopts ...pg.QOpt) (*LogPollerBlock, error) @@ -212,11 +212,28 @@ func (o *DbORM) SelectLatestLogByEventSigWithConfs(eventSig common.Hash, address return &l, nil } -// DeleteBlocksBefore delete all blocks before and including end. -func (o *DbORM) DeleteBlocksBefore(end int64, qopts ...pg.QOpt) error { +// DeleteBlocksBefore delete blocks before and including end. When limit is set, it will delete at most limit blocks. +// Otherwise, it will delete all blocks at once. +func (o *DbORM) DeleteBlocksBefore(end int64, limit int64, qopts ...pg.QOpt) (int64, error) { q := o.q.WithOpts(qopts...) - _, err := q.Exec(`DELETE FROM evm.log_poller_blocks WHERE block_number <= $1 AND evm_chain_id = $2`, end, ubig.New(o.chainID)) - return err + if limit > 0 { + return q.ExecQWithRowsAffected( + `DELETE FROM evm.log_poller_blocks + WHERE block_number IN ( + SELECT block_number FROM evm.log_poller_blocks + WHERE block_number <= $1 + AND evm_chain_id = $2 + LIMIT $3 + ) + AND evm_chain_id = $2`, + end, ubig.New(o.chainID), limit, + ) + } + return q.ExecQWithRowsAffected( + `DELETE FROM evm.log_poller_blocks + WHERE block_number <= $1 AND evm_chain_id = $2`, + end, ubig.New(o.chainID), + ) } func (o *DbORM) DeleteLogsAndBlocksAfter(start int64, qopts ...pg.QOpt) error { @@ -263,11 +280,30 @@ type Exp struct { ShouldDelete bool } -func (o *DbORM) DeleteExpiredLogs(qopts ...pg.QOpt) error { +func (o *DbORM) DeleteExpiredLogs(limit int64, qopts ...pg.QOpt) (int64, error) { qopts = append(qopts, pg.WithLongQueryTimeout()) q := o.q.WithOpts(qopts...) - return q.ExecQ(`WITH r AS + if limit > 0 { + return q.ExecQWithRowsAffected(` + DELETE FROM evm.logs + WHERE (evm_chain_id, address, event_sig, block_number) IN ( + SELECT l.evm_chain_id, l.address, l.event_sig, l.block_number + FROM evm.logs l + INNER JOIN ( + SELECT address, event, MAX(retention) AS retention + FROM evm.log_poller_filters + WHERE evm_chain_id = $1 + GROUP BY evm_chain_id, address, event + HAVING NOT 0 = ANY(ARRAY_AGG(retention)) + ) r ON l.evm_chain_id = $1 AND l.address = r.address AND l.event_sig = r.event + AND l.block_timestamp <= STATEMENT_TIMESTAMP() - (r.retention / 10^9 * interval '1 second') + LIMIT $2 + )`, + ubig.New(o.chainID), limit) + } + + return q.ExecQWithRowsAffected(`WITH r AS ( SELECT address, event, MAX(retention) AS retention FROM evm.log_poller_filters WHERE evm_chain_id=$1 GROUP BY evm_chain_id,address, event HAVING NOT 0 = ANY(ARRAY_AGG(retention)) diff --git a/core/chains/evm/logpoller/orm_test.go b/core/chains/evm/logpoller/orm_test.go index 9824d0e9426..8f89a237fd4 100644 --- a/core/chains/evm/logpoller/orm_test.go +++ b/core/chains/evm/logpoller/orm_test.go @@ -433,8 +433,9 @@ func TestORM(t *testing.T) { // Delete expired logs time.Sleep(2 * time.Millisecond) // just in case we haven't reached the end of the 1ms retention period - err = o1.DeleteExpiredLogs(pg.WithParentCtx(testutils.Context(t))) + deleted, err := o1.DeleteExpiredLogs(0, pg.WithParentCtx(testutils.Context(t))) require.NoError(t, err) + assert.Equal(t, int64(1), deleted) logs, err = o1.SelectLogsByBlockRange(1, latest.BlockNumber) require.NoError(t, err) // The only log which should be deleted is the one which matches filter1 (ret=1ms) but not filter12 (ret=1 hour) @@ -851,9 +852,11 @@ func TestORM_DeleteBlocksBefore(t *testing.T) { o1 := th.ORM require.NoError(t, o1.InsertBlock(common.HexToHash("0x1234"), 1, time.Now(), 0)) require.NoError(t, o1.InsertBlock(common.HexToHash("0x1235"), 2, time.Now(), 0)) - require.NoError(t, o1.DeleteBlocksBefore(1)) + deleted, err := o1.DeleteBlocksBefore(1, 0) + require.NoError(t, err) + assert.Equal(t, int64(1), deleted) // 1 should be gone. - _, err := o1.SelectBlockByNumber(1) + _, err = o1.SelectBlockByNumber(1) require.Equal(t, err, sql.ErrNoRows) b, err := o1.SelectBlockByNumber(2) require.NoError(t, err) @@ -861,7 +864,9 @@ func TestORM_DeleteBlocksBefore(t *testing.T) { // Clear multiple require.NoError(t, o1.InsertBlock(common.HexToHash("0x1236"), 3, time.Now(), 0)) require.NoError(t, o1.InsertBlock(common.HexToHash("0x1237"), 4, time.Now(), 0)) - require.NoError(t, o1.DeleteBlocksBefore(3)) + deleted, err = o1.DeleteBlocksBefore(3, 0) + require.NoError(t, err) + assert.Equal(t, int64(2), deleted) _, err = o1.SelectBlockByNumber(2) require.Equal(t, err, sql.ErrNoRows) _, err = o1.SelectBlockByNumber(3) @@ -1661,3 +1666,57 @@ func Benchmark_LogsDataWordBetween(b *testing.B) { assert.Len(b, logs, 1) } } + +func Benchmark_DeleteExpiredLogs(b *testing.B) { + chainId := big.NewInt(137) + _, db := heavyweight.FullTestDBV2(b, nil) + o := logpoller.NewORM(chainId, db, logger.Test(b), pgtest.NewQConfig(false)) + + numberOfReports := 200_000 + commitStoreAddress := utils.RandomAddress() + commitReportAccepted := utils.RandomBytes32() + + past := time.Now().Add(-1 * time.Hour) + + err := o.InsertFilter(logpoller.Filter{ + Name: "test filter", + EventSigs: []common.Hash{commitReportAccepted}, + Addresses: []common.Address{commitStoreAddress}, + Retention: 1 * time.Millisecond, + }) + require.NoError(b, err) + + for j := 0; j < 5; j++ { + var dbLogs []logpoller.Log + for i := 0; i < numberOfReports; i++ { + + dbLogs = append(dbLogs, logpoller.Log{ + EvmChainId: ubig.New(chainId), + LogIndex: int64(i + 1), + BlockHash: utils.RandomBytes32(), + BlockNumber: int64(i + 1), + BlockTimestamp: past, + EventSig: commitReportAccepted, + Topics: [][]byte{}, + Address: commitStoreAddress, + TxHash: utils.RandomHash(), + Data: []byte{}, + CreatedAt: past, + }) + } + require.NoError(b, o.InsertLogs(dbLogs)) + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + tx, err1 := db.Beginx() + assert.NoError(b, err1) + + _, err1 = o.DeleteExpiredLogs(0, pg.WithQueryer(tx)) + assert.NoError(b, err1) + + err1 = tx.Rollback() + assert.NoError(b, err1) + } +} diff --git a/core/chains/evm/txmgr/txmgr_test.go b/core/chains/evm/txmgr/txmgr_test.go index 0e28f2948ee..85e37571b5a 100644 --- a/core/chains/evm/txmgr/txmgr_test.go +++ b/core/chains/evm/txmgr/txmgr_test.go @@ -50,7 +50,7 @@ import ( func makeTestEvmTxm( t *testing.T, db *sqlx.DB, ethClient evmclient.Client, estimator gas.EvmFeeEstimator, ccfg txmgr.ChainConfig, fcfg txmgr.FeeConfig, txConfig evmconfig.Transactions, dbConfig txmgr.DatabaseConfig, listenerConfig txmgr.ListenerConfig, keyStore keystore.Eth) (txmgr.TxManager, error) { lggr := logger.Test(t) - lp := logpoller.NewLogPoller(logpoller.NewORM(testutils.FixtureChainID, db, lggr, pgtest.NewQConfig(true)), ethClient, lggr, 100*time.Millisecond, false, 2, 3, 2, 1000) + lp := logpoller.NewLogPoller(logpoller.NewORM(testutils.FixtureChainID, db, lggr, pgtest.NewQConfig(true)), ethClient, lggr, 100*time.Millisecond, false, 2, 3, 2, 1000, 0) // logic for building components (from evm/evm_txm.go) ------- lggr.Infow("Initializing EVM transaction manager", diff --git a/core/chains/legacyevm/chain.go b/core/chains/legacyevm/chain.go index 92936299cdb..66907b8352f 100644 --- a/core/chains/legacyevm/chain.go +++ b/core/chains/legacyevm/chain.go @@ -251,7 +251,8 @@ func newChain(ctx context.Context, cfg *evmconfig.ChainScoped, nodes []*toml.Nod int64(cfg.EVM().FinalityDepth()), int64(cfg.EVM().LogBackfillBatchSize()), int64(cfg.EVM().RPCDefaultBatchSize()), - int64(cfg.EVM().LogKeepBlocksDepth())) + int64(cfg.EVM().LogKeepBlocksDepth()), + int64(cfg.EVM().LogPrunePageSize())) } } diff --git a/core/config/docs/chains-evm.toml b/core/config/docs/chains-evm.toml index 6f2322fd6db..eec580a8e69 100644 --- a/core/config/docs/chains-evm.toml +++ b/core/config/docs/chains-evm.toml @@ -54,6 +54,9 @@ LogPollInterval = '15s' # Default # **ADVANCED** # LogKeepBlocksDepth works in conjunction with Feature.LogPoller. Controls how many blocks the poller will keep, must be greater than FinalityDepth+1. LogKeepBlocksDepth = 100000 # Default +# **ADVANCED** +# LogPrunePageSize defines size of the page for pruning logs. Controls how many logs/blocks (at most) are deleted in a single prune tick. Default value 0 means no paging, delete everything at once. +LogPrunePageSize = 0 # Default # MinContractPayment is the minimum payment in LINK required to execute a direct request job. This can be overridden on a per-job basis. MinContractPayment = '10000000000000 juels' # Default # MinIncomingConfirmations is the minimum required confirmations before a log event will be consumed. diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index 552a91d6e2f..13e18145e73 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -529,6 +529,7 @@ func TestConfig_Marshal(t *testing.T) { LogBackfillBatchSize: ptr[uint32](17), LogPollInterval: &minute, LogKeepBlocksDepth: ptr[uint32](100000), + LogPrunePageSize: ptr[uint32](0), MinContractPayment: commonassets.NewLinkFromJuels(math.MaxInt64), MinIncomingConfirmations: ptr[uint32](13), NonceAutoSync: ptr(true), @@ -923,6 +924,7 @@ LinkContractAddress = '0x538aAaB4ea120b2bC2fe5D296852D948F07D849e' LogBackfillBatchSize = 17 LogPollInterval = '1m0s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 13 MinContractPayment = '9.223372036854775807 link' NonceAutoSync = true diff --git a/core/services/chainlink/testdata/config-full.toml b/core/services/chainlink/testdata/config-full.toml index bc1b124ccb6..fc47602ef3b 100644 --- a/core/services/chainlink/testdata/config-full.toml +++ b/core/services/chainlink/testdata/config-full.toml @@ -253,6 +253,7 @@ LinkContractAddress = '0x538aAaB4ea120b2bC2fe5D296852D948F07D849e' LogBackfillBatchSize = 17 LogPollInterval = '1m0s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 13 MinContractPayment = '9.223372036854775807 link' NonceAutoSync = true diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml index 03990b02a50..2ad6bf30c50 100644 --- a/core/services/chainlink/testdata/config-multi-chain-effective.toml +++ b/core/services/chainlink/testdata/config-multi-chain-effective.toml @@ -240,6 +240,7 @@ LinkContractAddress = '0x514910771AF9Ca656af840dff83E8264EcF986CA' LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.1 link' NonceAutoSync = true @@ -327,6 +328,7 @@ LinkContractAddress = '0xa36085F69e2889c224210F603D836748e7dC0088' LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.1 link' NonceAutoSync = true @@ -409,6 +411,7 @@ LinkContractAddress = '0xb0897686c545045aFc77CF20eC7A532E3120E0F1' LogBackfillBatchSize = 1000 LogPollInterval = '1s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 5 MinContractPayment = '0.00001 link' NonceAutoSync = true diff --git a/core/services/ocr2/plugins/llo/onchain_channel_definition_cache_integration_test.go b/core/services/ocr2/plugins/llo/onchain_channel_definition_cache_integration_test.go index a9ba09d96bb..427dd6b32c2 100644 --- a/core/services/ocr2/plugins/llo/onchain_channel_definition_cache_integration_test.go +++ b/core/services/ocr2/plugins/llo/onchain_channel_definition_cache_integration_test.go @@ -77,7 +77,7 @@ func Test_ChannelDefinitionCache_Integration(t *testing.T) { require.NoError(t, err) t.Run("with zero fromblock", func(t *testing.T) { - lp := logpoller.NewLogPoller(logpoller.NewORM(testutils.SimulatedChainID, db, lggr, pgtest.NewQConfig(true)), ethClient, lggr, 100*time.Millisecond, false, 1, 3, 2, 1000) + lp := logpoller.NewLogPoller(logpoller.NewORM(testutils.SimulatedChainID, db, lggr, pgtest.NewQConfig(true)), ethClient, lggr, 100*time.Millisecond, false, 1, 3, 2, 1000, 0) servicetest.Run(t, lp) cdc := llo.NewChannelDefinitionCache(lggr, orm, lp, configStoreAddress, 0) @@ -142,7 +142,7 @@ func Test_ChannelDefinitionCache_Integration(t *testing.T) { t.Run("loads from ORM", func(t *testing.T) { // Override logpoller to always return no logs lp := &mockLogPoller{ - LogPoller: logpoller.NewLogPoller(logpoller.NewORM(testutils.SimulatedChainID, db, lggr, pgtest.NewQConfig(true)), ethClient, lggr, 100*time.Millisecond, false, 1, 3, 2, 1000), + LogPoller: logpoller.NewLogPoller(logpoller.NewORM(testutils.SimulatedChainID, db, lggr, pgtest.NewQConfig(true)), ethClient, lggr, 100*time.Millisecond, false, 1, 3, 2, 1000, 0), LatestBlockFn: func(qopts ...pg.QOpt) (int64, error) { return 0, nil }, @@ -176,7 +176,7 @@ func Test_ChannelDefinitionCache_Integration(t *testing.T) { pgtest.MustExec(t, db, `DELETE FROM channel_definitions`) t.Run("with non-zero fromBlock", func(t *testing.T) { - lp := logpoller.NewLogPoller(logpoller.NewORM(testutils.SimulatedChainID, db, lggr, pgtest.NewQConfig(true)), ethClient, lggr, 100*time.Millisecond, false, 1, 3, 2, 1000) + lp := logpoller.NewLogPoller(logpoller.NewORM(testutils.SimulatedChainID, db, lggr, pgtest.NewQConfig(true)), ethClient, lggr, 100*time.Millisecond, false, 1, 3, 2, 1000, 0) servicetest.Run(t, lp) cdc := llo.NewChannelDefinitionCache(lggr, orm, lp, configStoreAddress, channel2Block.Number().Int64()+1) diff --git a/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/integration_test.go b/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/integration_test.go index 5ef06f1bd08..ba89c52c2ef 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/integration_test.go +++ b/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/integration_test.go @@ -661,7 +661,7 @@ func setupDependencies(t *testing.T, db *sqlx.DB, backend *backends.SimulatedBac pollerLggr := logger.TestLogger(t) pollerLggr.SetLogLevel(zapcore.WarnLevel) lorm := logpoller.NewORM(big.NewInt(1337), db, pollerLggr, pgtest.NewQConfig(false)) - lp := logpoller.NewLogPoller(lorm, ethClient, pollerLggr, 100*time.Millisecond, false, 1, 2, 2, 1000) + lp := logpoller.NewLogPoller(lorm, ethClient, pollerLggr, 100*time.Millisecond, false, 1, 2, 2, 1000, 0) return lp, ethClient } diff --git a/core/services/pg/q.go b/core/services/pg/q.go index 49a18817de9..52225ac6168 100644 --- a/core/services/pg/q.go +++ b/core/services/pg/q.go @@ -199,6 +199,16 @@ func (q Q) ExecQIter(query string, args ...interface{}) (sql.Result, context.Can res, err := q.Queryer.ExecContext(ctx, query, args...) return res, cancel, ql.withLogError(err) } +func (q Q) ExecQWithRowsAffected(query string, args ...interface{}) (int64, error) { + res, cancel, err := q.ExecQIter(query, args...) + defer cancel() + if err != nil { + return 0, err + } + + rowsDeleted, err := res.RowsAffected() + return rowsDeleted, err +} func (q Q) ExecQ(query string, args ...interface{}) error { ctx, cancel := q.Context() defer cancel() diff --git a/core/services/pg/q_test.go b/core/services/pg/q_test.go index 6e59e499887..66258fabff5 100644 --- a/core/services/pg/q_test.go +++ b/core/services/pg/q_test.go @@ -3,8 +3,14 @@ package pg import ( "testing" + "github.com/google/uuid" + "github.com/jmoiron/sqlx" "github.com/lib/pq" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/store/dialects" ) func Test_sprintQ(t *testing.T) { @@ -51,3 +57,27 @@ func Test_sprintQ(t *testing.T) { }) } } + +func Test_ExecQWithRowsAffected(t *testing.T) { + db, err := sqlx.Open(string(dialects.TransactionWrappedPostgres), uuid.New().String()) + require.NoError(t, err) + q := NewQ(db, logger.NullLogger, NewQConfig(false)) + + require.NoError(t, q.ExecQ("CREATE TABLE testtable (a TEXT, b TEXT)")) + + rows, err := q.ExecQWithRowsAffected("INSERT INTO testtable (a, b) VALUES ($1, $2)", "foo", "bar") + require.NoError(t, err) + assert.Equal(t, int64(1), rows) + + rows, err = q.ExecQWithRowsAffected("INSERT INTO testtable (a, b) VALUES ($1, $1), ($2, $2), ($1, $2)", "foo", "bar") + require.NoError(t, err) + assert.Equal(t, int64(3), rows) + + rows, err = q.ExecQWithRowsAffected("delete from testtable") + require.NoError(t, err) + assert.Equal(t, int64(4), rows) + + rows, err = q.ExecQWithRowsAffected("delete from testtable") + require.NoError(t, err) + assert.Equal(t, int64(0), rows) +} diff --git a/core/services/promreporter/prom_reporter_test.go b/core/services/promreporter/prom_reporter_test.go index 60d6d9388fa..d283cb8f873 100644 --- a/core/services/promreporter/prom_reporter_test.go +++ b/core/services/promreporter/prom_reporter_test.go @@ -38,7 +38,7 @@ func newLegacyChainContainer(t *testing.T, db *sqlx.DB) legacyevm.LegacyChainCon ethClient := evmtest.NewEthClientMockWithDefaultChain(t) estimator := gas.NewEstimator(logger.TestLogger(t), ethClient, config, evmConfig.GasEstimator()) lggr := logger.TestLogger(t) - lp := logpoller.NewLogPoller(logpoller.NewORM(testutils.FixtureChainID, db, lggr, pgtest.NewQConfig(true)), ethClient, lggr, 100*time.Millisecond, false, 2, 3, 2, 1000) + lp := logpoller.NewLogPoller(logpoller.NewORM(testutils.FixtureChainID, db, lggr, pgtest.NewQConfig(true)), ethClient, lggr, 100*time.Millisecond, false, 2, 3, 2, 1000, 0) txm, err := txmgr.NewTxm( db, diff --git a/core/services/relay/evm/chain_reader_test.go b/core/services/relay/evm/chain_reader_test.go index c5fe0a16ed4..64d9f9f1cac 100644 --- a/core/services/relay/evm/chain_reader_test.go +++ b/core/services/relay/evm/chain_reader_test.go @@ -263,7 +263,7 @@ func (it *chainReaderInterfaceTester) GetChainReader(t *testing.T) clcommontypes lggr := logger.NullLogger db := pgtest.NewSqlxDB(t) - lp := logpoller.NewLogPoller(logpoller.NewORM(testutils.SimulatedChainID, db, lggr, pgtest.NewQConfig(true)), it.chain.Client(), lggr, time.Millisecond, false, 0, 1, 1, 10000) + lp := logpoller.NewLogPoller(logpoller.NewORM(testutils.SimulatedChainID, db, lggr, pgtest.NewQConfig(true)), it.chain.Client(), lggr, time.Millisecond, false, 0, 1, 1, 10000, 0) require.NoError(t, lp.Start(ctx)) it.chain.On("LogPoller").Return(lp) cr, err := evm.NewChainReaderService(lggr, lp, it.chain, it.chainConfig) diff --git a/core/services/relay/evm/config_poller_test.go b/core/services/relay/evm/config_poller_test.go index cd66e5479bf..089db6decd5 100644 --- a/core/services/relay/evm/config_poller_test.go +++ b/core/services/relay/evm/config_poller_test.go @@ -28,6 +28,7 @@ import ( ocrtypes2 "github.com/smartcontractkit/libocr/offchainreporting2plus/types" "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" evmclient "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" evmClientMocks "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client/mocks" @@ -89,7 +90,7 @@ func TestConfigPoller(t *testing.T) { cfg := pgtest.NewQConfig(false) ethClient = evmclient.NewSimulatedBackendClient(t, b, testutils.SimulatedChainID) lorm := logpoller.NewORM(testutils.SimulatedChainID, db, lggr, cfg) - lp = logpoller.NewLogPoller(lorm, ethClient, lggr, 100*time.Millisecond, false, 1, 2, 2, 1000) + lp = logpoller.NewLogPoller(lorm, ethClient, lggr, 100*time.Millisecond, false, 1, 2, 2, 1000, 0) servicetest.Run(t, lp) } diff --git a/core/services/relay/evm/functions/config_poller_test.go b/core/services/relay/evm/functions/config_poller_test.go index 2cf373d2e86..6a8c682a81b 100644 --- a/core/services/relay/evm/functions/config_poller_test.go +++ b/core/services/relay/evm/functions/config_poller_test.go @@ -81,7 +81,7 @@ func runTest(t *testing.T, pluginType functions.FunctionsPluginType, expectedDig defer ethClient.Close() lggr := logger.TestLogger(t) lorm := logpoller.NewORM(big.NewInt(1337), db, lggr, cfg) - lp := logpoller.NewLogPoller(lorm, ethClient, lggr, 100*time.Millisecond, false, 1, 2, 2, 1000) + lp := logpoller.NewLogPoller(lorm, ethClient, lggr, 100*time.Millisecond, false, 1, 2, 2, 1000, 0) servicetest.Run(t, lp) configPoller, err := functions.NewFunctionsConfigPoller(pluginType, lp, lggr) require.NoError(t, err) diff --git a/core/services/relay/evm/mercury/helpers_test.go b/core/services/relay/evm/mercury/helpers_test.go index f1686ee00c8..8283e80916e 100644 --- a/core/services/relay/evm/mercury/helpers_test.go +++ b/core/services/relay/evm/mercury/helpers_test.go @@ -167,7 +167,7 @@ func SetupTH(t *testing.T, feedID common.Hash) TestHarness { ethClient := evmclient.NewSimulatedBackendClient(t, b, big.NewInt(1337)) lggr := logger.TestLogger(t) lorm := logpoller.NewORM(big.NewInt(1337), db, lggr, cfg) - lp := logpoller.NewLogPoller(lorm, ethClient, lggr, 100*time.Millisecond, false, 1, 2, 2, 1000) + lp := logpoller.NewLogPoller(lorm, ethClient, lggr, 100*time.Millisecond, false, 1, 2, 2, 1000, 0) servicetest.Run(t, lp) configPoller, err := NewConfigPoller(lggr, lp, verifierAddress, feedID) diff --git a/core/services/vrf/v2/listener_v2_log_listener_test.go b/core/services/vrf/v2/listener_v2_log_listener_test.go index 6f5177c230a..c92795f55a6 100644 --- a/core/services/vrf/v2/listener_v2_log_listener_test.go +++ b/core/services/vrf/v2/listener_v2_log_listener_test.go @@ -92,7 +92,7 @@ func setupVRFLogPollerListenerTH(t *testing.T, // Poll period doesn't matter, we intend to call poll and save logs directly in the test. // Set it to some insanely high value to not interfere with any tests. - lp := logpoller.NewLogPoller(o, esc, lggr, 1*time.Hour, useFinalityTag, finalityDepth, backfillBatchSize, rpcBatchSize, keepFinalizedBlocksDepth) + lp := logpoller.NewLogPoller(o, esc, lggr, 1*time.Hour, useFinalityTag, finalityDepth, backfillBatchSize, rpcBatchSize, keepFinalizedBlocksDepth, 0) emitterAddress1, _, emitter1, err := log_emitter.DeployLogEmitter(owner, ec) require.NoError(t, err) diff --git a/core/web/resolver/chain_test.go b/core/web/resolver/chain_test.go index b7a4b7c8386..e2663af561f 100644 --- a/core/web/resolver/chain_test.go +++ b/core/web/resolver/chain_test.go @@ -10,6 +10,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/loop" commontypes "github.com/smartcontractkit/chainlink-common/pkg/types" + evmtoml "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/toml" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils/big" chainlinkmocks "github.com/smartcontractkit/chainlink/v2/core/services/chainlink/mocks" @@ -44,6 +45,7 @@ LinkContractAddress = '0x538aAaB4ea120b2bC2fe5D296852D948F07D849e' LogBackfillBatchSize = 17 LogPollInterval = '1m0s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 13 MinContractPayment = '9.223372036854775807 link' NonceAutoSync = true @@ -161,6 +163,7 @@ LinkContractAddress = '0x538aAaB4ea120b2bC2fe5D296852D948F07D849e' LogBackfillBatchSize = 17 LogPollInterval = '1m0s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 13 MinContractPayment = '9.223372036854775807 link' NonceAutoSync = true diff --git a/core/web/resolver/testdata/config-full.toml b/core/web/resolver/testdata/config-full.toml index 67ddbb33efd..2c84024cde6 100644 --- a/core/web/resolver/testdata/config-full.toml +++ b/core/web/resolver/testdata/config-full.toml @@ -253,6 +253,7 @@ LinkContractAddress = '0x538aAaB4ea120b2bC2fe5D296852D948F07D849e' LogBackfillBatchSize = 17 LogPollInterval = '1m0s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 13 MinContractPayment = '9.223372036854775807 link' NonceAutoSync = true diff --git a/core/web/resolver/testdata/config-multi-chain-effective.toml b/core/web/resolver/testdata/config-multi-chain-effective.toml index 03990b02a50..2ad6bf30c50 100644 --- a/core/web/resolver/testdata/config-multi-chain-effective.toml +++ b/core/web/resolver/testdata/config-multi-chain-effective.toml @@ -240,6 +240,7 @@ LinkContractAddress = '0x514910771AF9Ca656af840dff83E8264EcF986CA' LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.1 link' NonceAutoSync = true @@ -327,6 +328,7 @@ LinkContractAddress = '0xa36085F69e2889c224210F603D836748e7dC0088' LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.1 link' NonceAutoSync = true @@ -409,6 +411,7 @@ LinkContractAddress = '0xb0897686c545045aFc77CF20eC7A532E3120E0F1' LogBackfillBatchSize = 1000 LogPollInterval = '1s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 5 MinContractPayment = '0.00001 link' NonceAutoSync = true diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 6647dcc1efd..7ef27305809 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Gas bumping logic to the `SuggestedPriceEstimator`. The bumping mechanism for this estimator refetches the price from the RPC and adds a buffer on top using the greater of `BumpPercent` and `BumpMin`. - Add preliminary support for "llo" job type (Data Streams V1) +- Add `LogPrunePageSize` parameter to the EVM configuration. This parameter controls the number of logs removed during prune phase in LogPoller. Default value is 0, which deletes all logs at once - exactly how it used to work, so it doesn't require any changes on the product's side. ### Fixed diff --git a/docs/CONFIG.md b/docs/CONFIG.md index cb1653c1020..baafab0d347 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -1603,6 +1603,7 @@ LinkContractAddress = '0x514910771AF9Ca656af840dff83E8264EcF986CA' LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.1 link' NonceAutoSync = true @@ -1685,6 +1686,7 @@ LinkContractAddress = '0x20fE562d797A42Dcb3399062AE9546cd06f63280' LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.1 link' NonceAutoSync = true @@ -1766,6 +1768,7 @@ LinkContractAddress = '0x01BE23585060835E02B77ef475b0Cc51aA1e0709' LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.1 link' NonceAutoSync = true @@ -1847,6 +1850,7 @@ LinkContractAddress = '0x326C977E6efc84E512bB9C30f76E30c160eD06FB' LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.1 link' NonceAutoSync = true @@ -1929,6 +1933,7 @@ LinkContractAddress = '0x350a791Bfc2C21F9Ed5d10980Dad2e2638ffa7f6' LogBackfillBatchSize = 1000 LogPollInterval = '2s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -2010,6 +2015,7 @@ LinkContractAddress = '0x14AdaE34beF7ca957Ce2dDe5ADD97ea050123827' LogBackfillBatchSize = 1000 LogPollInterval = '30s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.001 link' NonceAutoSync = true @@ -2091,6 +2097,7 @@ LinkContractAddress = '0x8bBbd80981FE76d44854D8DF305e8985c19f0e78' LogBackfillBatchSize = 1000 LogPollInterval = '30s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.001 link' NonceAutoSync = true @@ -2172,6 +2179,7 @@ LinkContractAddress = '0xa36085F69e2889c224210F603D836748e7dC0088' LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.1 link' NonceAutoSync = true @@ -2254,6 +2262,7 @@ LinkContractAddress = '0x404460C6A5EdE2D891e8297795264fDe62ADBB75' LogBackfillBatchSize = 1000 LogPollInterval = '3s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -2334,6 +2343,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -2414,6 +2424,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -2495,6 +2506,7 @@ LinkContractAddress = '0x84b9B910527Ad5C03A9Ca831909E21e236EA7b06' LogBackfillBatchSize = 1000 LogPollInterval = '3s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -2577,6 +2589,7 @@ LinkContractAddress = '0xE2e73A1c69ecF83F464EFCE6A5be353a37cA09b2' LogBackfillBatchSize = 1000 LogPollInterval = '5s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -2658,6 +2671,7 @@ LinkContractAddress = '0x404460C6A5EdE2D891e8297795264fDe62ADBB75' LogBackfillBatchSize = 1000 LogPollInterval = '3s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -2739,6 +2753,7 @@ LinkContractAddress = '0xb0897686c545045aFc77CF20eC7A532E3120E0F1' LogBackfillBatchSize = 1000 LogPollInterval = '1s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 5 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -2820,6 +2835,7 @@ LinkContractAddress = '0x6F43FF82CCA38001B6699a8AC47A2d0E66939407' LogBackfillBatchSize = 1000 LogPollInterval = '1s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -2901,6 +2917,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '2s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -2982,6 +2999,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '5s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -3063,6 +3081,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '5s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -3145,6 +3164,7 @@ LinkContractAddress = '0xdc2CC710e42857672E7907CF474a69B63B93089f' LogBackfillBatchSize = 1000 LogPollInterval = '2s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -3226,6 +3246,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -3306,6 +3327,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -3387,6 +3409,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -3467,6 +3490,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '30s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -3548,6 +3572,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '3s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -3629,6 +3654,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '3s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -3709,6 +3735,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '100' NonceAutoSync = true @@ -3789,6 +3816,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '30s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -3870,6 +3898,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '2s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -3951,6 +3980,7 @@ LinkContractAddress = '0xfaFedb041c0DD4fA2Dc0d87a6B0979Ee6FA7af5F' LogBackfillBatchSize = 1000 LogPollInterval = '1s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -4031,6 +4061,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -4112,6 +4143,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '2s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -4193,6 +4225,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '5s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -4275,6 +4308,7 @@ LinkContractAddress = '0xf97f4df75117a78c1A5a0DBb814Af92458539FB4' LogBackfillBatchSize = 1000 LogPollInterval = '1s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -4356,6 +4390,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '5s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -4437,6 +4472,7 @@ LinkContractAddress = '0x0b9d5D9136855f6FEc3c0993feE6E9CE8a297846' LogBackfillBatchSize = 1000 LogPollInterval = '3s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -4518,6 +4554,7 @@ LinkContractAddress = '0x5947BB275c521040051D82396192181b413227A3' LogBackfillBatchSize = 1000 LogPollInterval = '3s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -4599,6 +4636,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '5s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -4679,6 +4717,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -4759,6 +4798,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -4840,6 +4880,7 @@ LinkContractAddress = '0x326C977E6efc84E512bB9C30f76E30c160eD06FB' LogBackfillBatchSize = 1000 LogPollInterval = '1s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 5 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -4921,6 +4962,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '2s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -5002,6 +5044,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '2s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -5084,6 +5127,7 @@ LinkContractAddress = '0x615fBe6372676474d9e6933d310469c9b68e9726' LogBackfillBatchSize = 1000 LogPollInterval = '1s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -5166,6 +5210,7 @@ LinkContractAddress = '0xd14838A68E8AFBAdE5efb411d5871ea0011AFd28' LogBackfillBatchSize = 1000 LogPollInterval = '1s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -5247,6 +5292,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '1s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -5328,6 +5374,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '3s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -5409,6 +5456,7 @@ FinalityTagEnabled = false LogBackfillBatchSize = 1000 LogPollInterval = '3s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -5490,6 +5538,7 @@ LinkContractAddress = '0x779877A7B0D9E8603169DdbD7836e478b4624789' LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.1 link' NonceAutoSync = true @@ -5571,6 +5620,7 @@ LinkContractAddress = '0x218532a12a389a4a92fC0C5Fb22901D1c19198aA' LogBackfillBatchSize = 1000 LogPollInterval = '2s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -5652,6 +5702,7 @@ LinkContractAddress = '0x8b12Ac23BFe11cAb03a634C1F117D64a7f2cFD3e' LogBackfillBatchSize = 1000 LogPollInterval = '2s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 1 MinContractPayment = '0.00001 link' NonceAutoSync = true @@ -5826,6 +5877,13 @@ LogKeepBlocksDepth = 100000 # Default ``` LogKeepBlocksDepth works in conjunction with Feature.LogPoller. Controls how many blocks the poller will keep, must be greater than FinalityDepth+1. +### LogPrunePageSize +:warning: **_ADVANCED_**: _Do not change this setting unless you know what you are doing._ +```toml +LogPrunePageSize = 0 # Default +``` +LogPrunePageSize defines size of the page for pruning logs. Controls how many logs/blocks (at most) are deleted in a single prune tick. Default value 0 means no paging, delete everything at once. + ### MinContractPayment ```toml MinContractPayment = '10000000000000 juels' # Default diff --git a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar index 648c94ed2a1..be61aef4898 100644 --- a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar @@ -296,6 +296,7 @@ LinkContractAddress = '0x514910771AF9Ca656af840dff83E8264EcF986CA' LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.1 link' NonceAutoSync = true diff --git a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar index dc7f87375a0..bf39b7d8394 100644 --- a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar @@ -296,6 +296,7 @@ LinkContractAddress = '0x514910771AF9Ca656af840dff83E8264EcF986CA' LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.1 link' NonceAutoSync = true diff --git a/testdata/scripts/node/validate/disk-based-logging.txtar b/testdata/scripts/node/validate/disk-based-logging.txtar index 25cba748c78..0ef1c6a7132 100644 --- a/testdata/scripts/node/validate/disk-based-logging.txtar +++ b/testdata/scripts/node/validate/disk-based-logging.txtar @@ -296,6 +296,7 @@ LinkContractAddress = '0x514910771AF9Ca656af840dff83E8264EcF986CA' LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.1 link' NonceAutoSync = true diff --git a/testdata/scripts/node/validate/invalid.txtar b/testdata/scripts/node/validate/invalid.txtar index 036d9544e74..e4eccf9895a 100644 --- a/testdata/scripts/node/validate/invalid.txtar +++ b/testdata/scripts/node/validate/invalid.txtar @@ -286,6 +286,7 @@ LinkContractAddress = '0x514910771AF9Ca656af840dff83E8264EcF986CA' LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.1 link' NonceAutoSync = true diff --git a/testdata/scripts/node/validate/valid.txtar b/testdata/scripts/node/validate/valid.txtar index 6a95692d295..9c1026d7f4c 100644 --- a/testdata/scripts/node/validate/valid.txtar +++ b/testdata/scripts/node/validate/valid.txtar @@ -293,6 +293,7 @@ LinkContractAddress = '0x514910771AF9Ca656af840dff83E8264EcF986CA' LogBackfillBatchSize = 1000 LogPollInterval = '15s' LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 MinIncomingConfirmations = 3 MinContractPayment = '0.1 link' NonceAutoSync = true