Using proper LogPrunePageSize

smartcontractkit · Mar 18, 2024 · fc8de5f · fc8de5f
1 parent 6c0f2b7
commit fc8de5f
Show file tree

Hide file tree

Showing 13 changed files with 73 additions and 71 deletions.
diff --git a/core/chains/evm/config/toml/defaults/fallback.toml b/core/chains/evm/config/toml/defaults/fallback.toml
@@ -6,7 +6,8 @@ FinalityTagEnabled = false
 LogBackfillBatchSize = 1000
 LogPollInterval = '15s'
 LogKeepBlocksDepth = 100000
-LogPrunePageSize = 0
+# CCIP uses paging when removing logs to avoid pushing too much presssure on the database
+LogPrunePageSize = 10000
 BackupLogPollerBlockDelay = 100
 MinContractPayment = '.00001 link'
 MinIncomingConfirmations = 3

diff --git a/core/config/docs/chains-evm.toml b/core/config/docs/chains-evm.toml
@@ -56,7 +56,7 @@ LogPollInterval = '15s' # Default
 LogKeepBlocksDepth = 100000 # Default
 # **ADVANCED**
 # LogPrunePageSize defines size of the page for pruning logs. Controls how many logs/blocks (at most) are deleted in a single prune tick. Default value 0 means no paging, delete everything at once.
-LogPrunePageSize = 0 # Default
+LogPrunePageSize = 10000 # Default
 # **ADVANCED**
 # BackupLogPollerBlockDelay works in conjunction with Feature.LogPoller. Controls the block delay of Backup LogPoller, affecting how far behind the latest finalized block it starts and how often it runs.
 # BackupLogPollerDelay=0 will disable Backup LogPoller (_not recommended for production environment_).

diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go
@@ -530,7 +530,7 @@ func TestConfig_Marshal(t *testing.T) {
 				LogBackfillBatchSize:      ptr[uint32](17),
 				LogPollInterval:           &minute,
 				LogKeepBlocksDepth:        ptr[uint32](100000),
-				LogPrunePageSize:          ptr[uint32](0),
+				LogPrunePageSize:          ptr[uint32](10000),
 				BackupLogPollerBlockDelay: ptr[uint64](532),
 				MinContractPayment:        commonassets.NewLinkFromJuels(math.MaxInt64),
 				MinIncomingConfirmations:  ptr[uint32](13),
@@ -928,7 +928,7 @@ LinkContractAddress = '0x538aAaB4ea120b2bC2fe5D296852D948F07D849e'
 LogBackfillBatchSize = 17
 LogPollInterval = '1m0s'
 LogKeepBlocksDepth = 100000
-LogPrunePageSize = 0
+LogPrunePageSize = 10000
 BackupLogPollerBlockDelay = 532
 MinIncomingConfirmations = 13
 MinContractPayment = '9.223372036854775807 link'

diff --git a/core/services/chainlink/testdata/config-full.toml b/core/services/chainlink/testdata/config-full.toml
@@ -254,7 +254,7 @@ LinkContractAddress = '0x538aAaB4ea120b2bC2fe5D296852D948F07D849e'
 LogBackfillBatchSize = 17
 LogPollInterval = '1m0s'
 LogKeepBlocksDepth = 100000
-LogPrunePageSize = 0
+LogPrunePageSize = 10000
 BackupLogPollerBlockDelay = 532
 MinIncomingConfirmations = 13
 MinContractPayment = '9.223372036854775807 link'

diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml
@@ -241,7 +241,7 @@ LinkContractAddress = '0x514910771AF9Ca656af840dff83E8264EcF986CA'
 LogBackfillBatchSize = 1000
 LogPollInterval = '15s'
 LogKeepBlocksDepth = 100000
-LogPrunePageSize = 0
+LogPrunePageSize = 10000
 BackupLogPollerBlockDelay = 100
 MinIncomingConfirmations = 3
 MinContractPayment = '0.1 link'
@@ -331,7 +331,7 @@ LinkContractAddress = '0xa36085F69e2889c224210F603D836748e7dC0088'
 LogBackfillBatchSize = 1000
 LogPollInterval = '15s'
 LogKeepBlocksDepth = 100000
-LogPrunePageSize = 0
+LogPrunePageSize = 10000
 BackupLogPollerBlockDelay = 100
 MinIncomingConfirmations = 3
 MinContractPayment = '0.1 link'
@@ -416,7 +416,7 @@ LinkContractAddress = '0xb0897686c545045aFc77CF20eC7A532E3120E0F1'
 LogBackfillBatchSize = 1000
 LogPollInterval = '1s'
 LogKeepBlocksDepth = 100000
-LogPrunePageSize = 0
+LogPrunePageSize = 10000
 BackupLogPollerBlockDelay = 100
 MinIncomingConfirmations = 5
 MinContractPayment = '0.00001 link'

diff --git a/core/web/resolver/testdata/config-multi-chain-effective.toml b/core/web/resolver/testdata/config-multi-chain-effective.toml
@@ -241,7 +241,7 @@ LinkContractAddress = '0x514910771AF9Ca656af840dff83E8264EcF986CA'
 LogBackfillBatchSize = 1000
 LogPollInterval = '15s'
 LogKeepBlocksDepth = 100000
-LogPrunePageSize = 0
+LogPrunePageSize = 10000
 BackupLogPollerBlockDelay = 100
 MinIncomingConfirmations = 3
 MinContractPayment = '0.1 link'

diff --git a/docs/CHANGELOG_CCIP.md b/docs/CHANGELOG_CCIP.md
@@ -13,7 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - Added optional `PriceReportingDisabled` to signal lane should not report gas and token prices.
 - `OffRamp` offchain config changed:
   - `MaxGasPrice` and `DestMaxGasPrice` will be ignored if specified. DestMaxGasPrice will be read from PriceMax in chain TOML.
-
+- All LogPoller's filters have now retention defined. This means that the LogPoller will remove logs older than the retention defined per filter. 
+  Additionally, logs are now removed from the database in batches, defined by `LogPrunePageSize` in the toml config
 
 ## 1.4.0 - 2024-02-16