From 8c50820231d417a49f9e30cf448e65caa55b2b77 Mon Sep 17 00:00:00 2001 From: simsonraj Date: Tue, 11 Jun 2024 00:08:14 +0530 Subject: [PATCH 01/14] Hedera config + errors --- core/chains/evm/client/errors.go | 11 +++++++- core/chains/evm/client/errors_test.go | 2 ++ .../config/toml/defaults/Hedera_Testnet.toml | 27 +++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 core/chains/evm/config/toml/defaults/Hedera_Testnet.toml diff --git a/core/chains/evm/client/errors.go b/core/chains/evm/client/errors.go index f43b5189c7e..16aa658a650 100644 --- a/core/chains/evm/client/errors.go +++ b/core/chains/evm/client/errors.go @@ -249,7 +249,16 @@ var zkEvm = ClientErrors{ OutOfCounters: regexp.MustCompile(`(?:: |^)not enough .* counters to continue the execution$`), } -var clients = []ClientErrors{parity, geth, arbitrum, metis, substrate, avalanche, nethermind, harmony, besu, erigon, klaytn, celo, zkSync, zkEvm} +var hederaFatal = regexp.MustCompile(`(: |^)(execution reverted)(:|$) | ^Transaction gas limit '(\d+)' exceeds block gas limit '(\d+)' | ^Transaction gas limit provided '(\d+)' is insufficient of intrinsic gas required '(\d+)' | ^Oversized data:`) +var hedera = ClientErrors{ + NonceTooLow: regexp.MustCompile(`Nonce too low`), + NonceTooHigh: regexp.MustCompile(`Nonce too high`), + TerminallyUnderpriced: regexp.MustCompile(`Gas price '(\d+)' is below configured minimum gas price '(\d+)'$`), + InsufficientEth: regexp.MustCompile(`insufficient funds for transfer$ | failed precheck with status INSUFFICIENT_PAYER_BALANCE`), + Fatal: hederaFatal, +} + +var clients = []ClientErrors{parity, geth, arbitrum, metis, substrate, avalanche, nethermind, harmony, besu, erigon, klaytn, celo, zkSync, zkEvm, hedera} // ClientErrorRegexes returns a map of compiled regexes for each error type func ClientErrorRegexes(errsRegex config.ClientErrors) *ClientErrors { diff --git a/core/chains/evm/client/errors_test.go b/core/chains/evm/client/errors_test.go index cca54c2a4a9..2ebd1171fec 100644 --- a/core/chains/evm/client/errors_test.go +++ b/core/chains/evm/client/errors_test.go @@ -166,6 +166,7 @@ func Test_Eth_Errors(t *testing.T) { {"max fee per gas less than block base fee", true, "zkSync"}, {"virtual machine entered unexpected state. please contact developers and provide transaction details that caused this error. Error description: The operator included transaction with an unacceptable gas price", true, "zkSync"}, {"client error terminally underpriced", true, "tomlConfig"}, + {"[Request ID: e4d09e44-19a4-4eb7-babe-270db4c2ebc9] Gas price '830000000000' is below configured minimum gas price '950000000000'", true, "hedera"}, } for _, test := range tests { @@ -214,6 +215,7 @@ func Test_Eth_Errors(t *testing.T) { {"insufficient funds for gas + value. balance: 42719769622667482000, fee: 48098250000000, value: 42719769622667482000", true, "celo"}, {"client error insufficient eth", true, "tomlConfig"}, {"transaction would cause overdraft", true, "Geth"}, + {"[Request ID: 9dd78806-58c8-4e6d-89a8-a60962abe705] Error invoking RPC: transaction 0.0.3041916@1717691931.680570179 failed precheck with status INSUFFICIENT_PAYER_BALANCE", true, "hedera"}, } for _, test := range tests { err = evmclient.NewSendErrorS(test.message) diff --git a/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml b/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml new file mode 100644 index 00000000000..07f6ef5f64e --- /dev/null +++ b/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml @@ -0,0 +1,27 @@ +ChainID = '296' +#considering the 3-5 (6 including a buffer) seconds of finality and 2 seconds block production +#we set the depth to 6/2 = 3 blocks +FinalityDepth = 3 +#Hedera has high TPS, so polling less often +LogPollInterval = '20s' +MinIncomingConfirmations = 1 +#turning off the liveness detection, since the Websocket connection is not reliable, can be turned on if needed +NoNewHeadsThreshold = '0' + +[BalanceMonitor] +Enabled = true + +[GasEstimator] +Mode = 'SuggestedPrice' +# since Hedera dont have mempool and there's no way for a node to front run or a user to bribe a node to submit the transaction earlier than it's consensus timestamp +# but they have automated congesting pricing throttling which would mean at high sustained level the gasPrice itself could be increased to prevent malicious behaviour. +# so wait for 3 blocks before bumping the gas price, the increased gas Price could be adjusted +BumpThreshold = 3 +BumpMin = '10 gwei' +BumpPercent = 20 + +[Transactions] +# to hit throttling you'd need to maintain 15 m gas /sec over a prolonged period of time. +# Because our block times are every 2 secs it's less less likely to happen as compared to other chains +# so setting a gap of 5 blocks to prevent sending transaction during throttling +ResendAfterThreshold = '10s' From 6cd388ca57e9266d52f375962150ec33823d6a0b Mon Sep 17 00:00:00 2001 From: simsonraj Date: Tue, 11 Jun 2024 00:09:15 +0530 Subject: [PATCH 02/14] docs update --- docs/CONFIG.md | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 07a26246d4e..eb6d54fa638 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -3530,6 +3530,95 @@ GasLimit = 5400000

+
Hedera Testnet (296)

+ +```toml +AutoCreateKey = true +BlockBackfillDepth = 10 +BlockBackfillSkip = false +FinalityDepth = 3 +FinalityTagEnabled = false +LogBackfillBatchSize = 1000 +LogPollInterval = '20s' +LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 +BackupLogPollerBlockDelay = 100 +MinIncomingConfirmations = 1 +MinContractPayment = '0.00001 link' +NonceAutoSync = true +NoNewHeadsThreshold = '0s' +RPCDefaultBatchSize = 250 +RPCBlockQueryDelay = 1 + +[Transactions] +ForwardersEnabled = false +MaxInFlight = 16 +MaxQueued = 250 +ReaperInterval = '1h0m0s' +ReaperThreshold = '168h0m0s' +ResendAfterThreshold = '10s' + +[Transactions.AutoPurge] +Enabled = false + +[BalanceMonitor] +Enabled = true + +[GasEstimator] +Mode = 'SuggestedPrice' +PriceDefault = '20 gwei' +PriceMax = '115792089237316195423570985008687907853269984665.640564039457584007913129639935 tether' +PriceMin = '1 gwei' +LimitDefault = 500000 +LimitMax = 500000 +LimitMultiplier = '1' +LimitTransfer = 21000 +BumpMin = '10 gwei' +BumpPercent = 20 +BumpThreshold = 3 +EIP1559DynamicFees = false +FeeCapDefault = '100 gwei' +TipCapDefault = '1 wei' +TipCapMin = '1 wei' + +[GasEstimator.BlockHistory] +BatchSize = 25 +BlockHistorySize = 8 +CheckInclusionBlocks = 12 +CheckInclusionPercentile = 90 +TransactionPercentile = 60 + +[HeadTracker] +HistoryDepth = 100 +MaxBufferSize = 3 +SamplingInterval = '1s' +MaxAllowedFinalityDepth = 10000 +FinalityTagBypass = true + +[NodePool] +PollFailureThreshold = 5 +PollInterval = '10s' +SelectionMode = 'HighestHead' +SyncThreshold = 5 +LeaseDuration = '0s' +NodeIsSyncingEnabled = false +FinalizedBlockPollInterval = '5s' + +[OCR] +ContractConfirmations = 4 +ContractTransmitterTransmitTimeout = '10s' +DatabaseTimeout = '10s' +DeltaCOverride = '168h0m0s' +DeltaCJitterOverride = '1h0m0s' +ObservationGracePeriod = '1s' + +[OCR2] +[OCR2.Automation] +GasLimit = 5400000 +``` + +

+
zkSync Sepolia (300)

```toml From 89d2c6369455446c69140aae71d637e4989e4923 Mon Sep 17 00:00:00 2001 From: simsonraj Date: Tue, 11 Jun 2024 14:29:41 +0530 Subject: [PATCH 03/14] Test cases --- core/chains/evm/client/errors_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/chains/evm/client/errors_test.go b/core/chains/evm/client/errors_test.go index 2ebd1171fec..b313fe61515 100644 --- a/core/chains/evm/client/errors_test.go +++ b/core/chains/evm/client/errors_test.go @@ -44,6 +44,7 @@ func Test_Eth_Errors(t *testing.T) { {"call failed: OldNonce, Current nonce: 22, nonce of rejected tx: 17", true, "Nethermind"}, {"nonce too low. allowed nonce range: 427 - 447, actual: 426", true, "zkSync"}, {"client error nonce too low", true, "tomlConfig"}, + {"[Request ID: 2e952947-ffad-408b-aed9-35f3ed152001] Nonce too low. Provided nonce: 15, current nonce: 15", true, "hedera"}, } for _, test := range tests { @@ -65,6 +66,7 @@ func Test_Eth_Errors(t *testing.T) { {"nonce too high", true, "Erigon"}, {"nonce too high. allowed nonce range: 427 - 477, actual: 527", true, "zkSync"}, {"client error nonce too high", true, "tomlConfig"}, + {"[Request ID: 3ec591b4-9396-49f4-a03f-06c415a7cc6a] Nonce too high. Provided nonce: 16, current nonce: 15", true, "hedera"}, } for _, test := range tests { From 61fd5f6493898ec820f69e7f41a079e0c2f40796 Mon Sep 17 00:00:00 2001 From: simsonraj Date: Tue, 11 Jun 2024 20:43:49 +0530 Subject: [PATCH 04/14] Hedera Mainnet config --- .../config/toml/defaults/Hedera_Mainnet.toml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 core/chains/evm/config/toml/defaults/Hedera_Mainnet.toml diff --git a/core/chains/evm/config/toml/defaults/Hedera_Mainnet.toml b/core/chains/evm/config/toml/defaults/Hedera_Mainnet.toml new file mode 100644 index 00000000000..7eac46e69f3 --- /dev/null +++ b/core/chains/evm/config/toml/defaults/Hedera_Mainnet.toml @@ -0,0 +1,27 @@ +ChainID = '295' +#considering the 3-5 (6 including a buffer) seconds of finality and 2 seconds block production +#we set the depth to 6/2 = 3 blocks +FinalityDepth = 3 +#Hedera has high TPS, so polling less often +LogPollInterval = '20s' +MinIncomingConfirmations = 1 +#turning off the liveness detection, since the Websocket connection is not reliable, can be turned on if needed +NoNewHeadsThreshold = '0' + +[BalanceMonitor] +Enabled = true + +[GasEstimator] +Mode = 'SuggestedPrice' +# since Hedera dont have mempool and there's no way for a node to front run or a user to bribe a node to submit the transaction earlier than it's consensus timestamp +# but they have automated congesting pricing throttling which would mean at high sustained level the gasPrice itself could be increased to prevent malicious behaviour. +# so wait for 3 blocks before bumping the gas price, the increased gas Price could be adjusted +BumpThreshold = 3 +BumpMin = '10 gwei' +BumpPercent = 20 + +[Transactions] +# to hit throttling you'd need to maintain 15 m gas /sec over a prolonged period of time. +# Because our block times are every 2 secs it's less less likely to happen as compared to other chains +# so setting a gap of 5 blocks to prevent sending transaction during throttling +ResendAfterThreshold = '10s' From b0abb19e99cd986a503368b680d3089a60415fb4 Mon Sep 17 00:00:00 2001 From: simsonraj Date: Tue, 11 Jun 2024 20:46:28 +0530 Subject: [PATCH 05/14] config update --- docs/CONFIG.md | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/docs/CONFIG.md b/docs/CONFIG.md index eb6d54fa638..a45a439e8e8 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -3530,6 +3530,95 @@ GasLimit = 5400000

+
Hedera Mainnet (295)

+ +```toml +AutoCreateKey = true +BlockBackfillDepth = 10 +BlockBackfillSkip = false +FinalityDepth = 3 +FinalityTagEnabled = false +LogBackfillBatchSize = 1000 +LogPollInterval = '20s' +LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 +BackupLogPollerBlockDelay = 100 +MinIncomingConfirmations = 1 +MinContractPayment = '0.00001 link' +NonceAutoSync = true +NoNewHeadsThreshold = '0s' +RPCDefaultBatchSize = 250 +RPCBlockQueryDelay = 1 + +[Transactions] +ForwardersEnabled = false +MaxInFlight = 16 +MaxQueued = 250 +ReaperInterval = '1h0m0s' +ReaperThreshold = '168h0m0s' +ResendAfterThreshold = '10s' + +[Transactions.AutoPurge] +Enabled = false + +[BalanceMonitor] +Enabled = true + +[GasEstimator] +Mode = 'SuggestedPrice' +PriceDefault = '20 gwei' +PriceMax = '115792089237316195423570985008687907853269984665.640564039457584007913129639935 tether' +PriceMin = '1 gwei' +LimitDefault = 500000 +LimitMax = 500000 +LimitMultiplier = '1' +LimitTransfer = 21000 +BumpMin = '10 gwei' +BumpPercent = 20 +BumpThreshold = 3 +EIP1559DynamicFees = false +FeeCapDefault = '100 gwei' +TipCapDefault = '1 wei' +TipCapMin = '1 wei' + +[GasEstimator.BlockHistory] +BatchSize = 25 +BlockHistorySize = 8 +CheckInclusionBlocks = 12 +CheckInclusionPercentile = 90 +TransactionPercentile = 60 + +[HeadTracker] +HistoryDepth = 100 +MaxBufferSize = 3 +SamplingInterval = '1s' +MaxAllowedFinalityDepth = 10000 +FinalityTagBypass = true + +[NodePool] +PollFailureThreshold = 5 +PollInterval = '10s' +SelectionMode = 'HighestHead' +SyncThreshold = 5 +LeaseDuration = '0s' +NodeIsSyncingEnabled = false +FinalizedBlockPollInterval = '5s' + +[OCR] +ContractConfirmations = 4 +ContractTransmitterTransmitTimeout = '10s' +DatabaseTimeout = '10s' +DeltaCOverride = '168h0m0s' +DeltaCJitterOverride = '1h0m0s' +ObservationGracePeriod = '1s' + +[OCR2] +[OCR2.Automation] +GasLimit = 5400000 +``` + +

+
Hedera Testnet (296)

```toml From 43a180871eb59e5d0ec269302b8830406950608a Mon Sep 17 00:00:00 2001 From: simsonraj Date: Wed, 12 Jun 2024 22:22:30 +0530 Subject: [PATCH 06/14] fixed error validation for Insufficient funds --- core/chains/evm/client/errors.go | 2 +- core/chains/evm/client/errors_test.go | 1 + core/chains/evm/config/toml/defaults/Hedera_Testnet.toml | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/core/chains/evm/client/errors.go b/core/chains/evm/client/errors.go index 16aa658a650..d24f084abc2 100644 --- a/core/chains/evm/client/errors.go +++ b/core/chains/evm/client/errors.go @@ -254,7 +254,7 @@ var hedera = ClientErrors{ NonceTooLow: regexp.MustCompile(`Nonce too low`), NonceTooHigh: regexp.MustCompile(`Nonce too high`), TerminallyUnderpriced: regexp.MustCompile(`Gas price '(\d+)' is below configured minimum gas price '(\d+)'$`), - InsufficientEth: regexp.MustCompile(`insufficient funds for transfer$ | failed precheck with status INSUFFICIENT_PAYER_BALANCE`), + InsufficientEth: regexp.MustCompile(`Insufficient funds for transfer| failed precheck with status INSUFFICIENT_PAYER_BALANCE`), Fatal: hederaFatal, } diff --git a/core/chains/evm/client/errors_test.go b/core/chains/evm/client/errors_test.go index b313fe61515..202ab4f3acc 100644 --- a/core/chains/evm/client/errors_test.go +++ b/core/chains/evm/client/errors_test.go @@ -218,6 +218,7 @@ func Test_Eth_Errors(t *testing.T) { {"client error insufficient eth", true, "tomlConfig"}, {"transaction would cause overdraft", true, "Geth"}, {"[Request ID: 9dd78806-58c8-4e6d-89a8-a60962abe705] Error invoking RPC: transaction 0.0.3041916@1717691931.680570179 failed precheck with status INSUFFICIENT_PAYER_BALANCE", true, "hedera"}, + {"[Request ID: 6198d2a3-590f-4724-aae5-69fecead0c49] Insufficient funds for transfer", true, "hedera"}, } for _, test := range tests { err = evmclient.NewSendErrorS(test.message) diff --git a/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml b/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml index 07f6ef5f64e..23d63ac5a7d 100644 --- a/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml +++ b/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml @@ -25,3 +25,8 @@ BumpPercent = 20 # Because our block times are every 2 secs it's less less likely to happen as compared to other chains # so setting a gap of 5 blocks to prevent sending transaction during throttling ResendAfterThreshold = '10s' + + +[NodePool] +#increasing to 3x the default polling interval to allow some buffer before decalring out-of-sync & since Hedera has high TPS +SyncThreshold = 30 \ No newline at end of file From d83539a5f4bd4407ac14d88fac7d3684a2d88be1 Mon Sep 17 00:00:00 2001 From: simsonraj Date: Wed, 12 Jun 2024 22:23:25 +0530 Subject: [PATCH 07/14] extended logging params for more info --- common/client/node_lifecycle.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/client/node_lifecycle.go b/common/client/node_lifecycle.go index 5947774e202..16d77de3a45 100644 --- a/common/client/node_lifecycle.go +++ b/common/client/node_lifecycle.go @@ -189,7 +189,8 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { _, num, td := n.StateAndLatest() if outOfSync, liveNodes := n.syncStatus(num, td); outOfSync { // note: there must be another live node for us to be out of sync - lggr.Errorw("RPC endpoint has fallen behind", "blockNumber", num, "totalDifficulty", td, "nodeState", n.State()) + _, highest, greatest := n.nLiveNodes() + lggr.Errorw("RPC endpoint has fallen behind", "blockNumber", num, "totalDifficulty", td, "nodeState", n.State(), "highest", highest, "greatest", greatest) if liveNodes < 2 { lggr.Criticalf("RPC endpoint has fallen behind; %s %s", msgCannotDisable, msgDegradedState) continue From 9b11b00cb1e8daf57209498b406a3b07da3bbd86 Mon Sep 17 00:00:00 2001 From: simsonraj Date: Wed, 12 Jun 2024 23:16:15 +0530 Subject: [PATCH 08/14] added errors & updated configs --- core/chains/evm/client/errors.go | 2 +- core/chains/evm/client/errors_test.go | 1 + core/chains/evm/config/toml/defaults/Hedera_Testnet.toml | 8 ++++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/core/chains/evm/client/errors.go b/core/chains/evm/client/errors.go index d24f084abc2..b00cd2263e9 100644 --- a/core/chains/evm/client/errors.go +++ b/core/chains/evm/client/errors.go @@ -249,7 +249,7 @@ var zkEvm = ClientErrors{ OutOfCounters: regexp.MustCompile(`(?:: |^)not enough .* counters to continue the execution$`), } -var hederaFatal = regexp.MustCompile(`(: |^)(execution reverted)(:|$) | ^Transaction gas limit '(\d+)' exceeds block gas limit '(\d+)' | ^Transaction gas limit provided '(\d+)' is insufficient of intrinsic gas required '(\d+)' | ^Oversized data:`) +var hederaFatal = regexp.MustCompile(`(: |^)(execution reverted)(:|$) | ^Transaction gas limit '(\d+)' exceeds block gas limit '(\d+)' | ^Transaction gas limit provided '(\d+)' is insufficient of intrinsic gas required '(\d+)' | ^Oversized data:|status INVALID_SIGNATURE`) var hedera = ClientErrors{ NonceTooLow: regexp.MustCompile(`Nonce too low`), NonceTooHigh: regexp.MustCompile(`Nonce too high`), diff --git a/core/chains/evm/client/errors_test.go b/core/chains/evm/client/errors_test.go index 202ab4f3acc..c7a2a590927 100644 --- a/core/chains/evm/client/errors_test.go +++ b/core/chains/evm/client/errors_test.go @@ -385,6 +385,7 @@ func Test_Eth_Errors_Fatal(t *testing.T) { {"Failed to serialize transaction: oversized data. max: 1000000; actual: 1000000", true, "zkSync"}, {"client error fatal", true, "tomlConfig"}, + {"[Request ID: d9711488-4c1e-4af2-bc1f-7969913d7b60] Error invoking RPC: transaction 0.0.4425573@1718213476.914320044 failed precheck with status INVALID_SIGNATURE", true, "hedera"}, } for _, test := range tests { diff --git a/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml b/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml index 23d63ac5a7d..f79773cc9e9 100644 --- a/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml +++ b/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml @@ -1,9 +1,9 @@ ChainID = '296' #considering the 3-5 (6 including a buffer) seconds of finality and 2 seconds block production -#we set the depth to 6/2 = 3 blocks -FinalityDepth = 3 +#we set the depth to 6/2 = 3 blocks, setting to 10 for safety +FinalityDepth = 10 #Hedera has high TPS, so polling less often -LogPollInterval = '20s' +LogPollInterval = '10s' MinIncomingConfirmations = 1 #turning off the liveness detection, since the Websocket connection is not reliable, can be turned on if needed NoNewHeadsThreshold = '0' @@ -29,4 +29,4 @@ ResendAfterThreshold = '10s' [NodePool] #increasing to 3x the default polling interval to allow some buffer before decalring out-of-sync & since Hedera has high TPS -SyncThreshold = 30 \ No newline at end of file +SyncThreshold = 0 \ No newline at end of file From 8941aeb3e2058eadd6fc660b15f7b11f64380732 Mon Sep 17 00:00:00 2001 From: simsonraj Date: Thu, 13 Jun 2024 17:25:11 +0530 Subject: [PATCH 09/14] another fix --- core/chains/evm/client/errors.go | 2 +- core/chains/evm/config/toml/defaults/Hedera_Testnet.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/chains/evm/client/errors.go b/core/chains/evm/client/errors.go index b00cd2263e9..b03febbaedc 100644 --- a/core/chains/evm/client/errors.go +++ b/core/chains/evm/client/errors.go @@ -253,7 +253,7 @@ var hederaFatal = regexp.MustCompile(`(: |^)(execution reverted)(:|$) | ^Transac var hedera = ClientErrors{ NonceTooLow: regexp.MustCompile(`Nonce too low`), NonceTooHigh: regexp.MustCompile(`Nonce too high`), - TerminallyUnderpriced: regexp.MustCompile(`Gas price '(\d+)' is below configured minimum gas price '(\d+)'$`), + TerminallyUnderpriced: regexp.MustCompile(`Gas price '(\d+)' is below configured minimum gas price '(\d+)'`), InsufficientEth: regexp.MustCompile(`Insufficient funds for transfer| failed precheck with status INSUFFICIENT_PAYER_BALANCE`), Fatal: hederaFatal, } diff --git a/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml b/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml index f79773cc9e9..7d7ff0ea561 100644 --- a/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml +++ b/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml @@ -28,5 +28,5 @@ ResendAfterThreshold = '10s' [NodePool] -#increasing to 3x the default polling interval to allow some buffer before decalring out-of-sync & since Hedera has high TPS +#Disabling sync threshold check for testing SyncThreshold = 0 \ No newline at end of file From 21b25bb3d6f4db28a2c075ef495de56f57e176ad Mon Sep 17 00:00:00 2001 From: simsonraj Date: Thu, 13 Jun 2024 23:43:34 +0530 Subject: [PATCH 10/14] updated the errors --- core/chains/evm/client/errors.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/chains/evm/client/errors.go b/core/chains/evm/client/errors.go index b03febbaedc..a2d67b939de 100644 --- a/core/chains/evm/client/errors.go +++ b/core/chains/evm/client/errors.go @@ -70,6 +70,8 @@ func (e *ClientErrors) ErrIs(err error, errorTypes ...int) bool { if err == nil { return false } + + cause := pkgerrors.Cause(err).Error() for _, errorType := range errorTypes { if _, ok := (*e)[errorType]; !ok { return false @@ -77,7 +79,7 @@ func (e *ClientErrors) ErrIs(err error, errorTypes ...int) bool { if (*e)[errorType].String() == "" { return false } - if (*e)[errorType].MatchString(pkgerrors.Cause(err).Error()) { + if (*e)[errorType].MatchString(cause) { return true } } @@ -253,7 +255,7 @@ var hederaFatal = regexp.MustCompile(`(: |^)(execution reverted)(:|$) | ^Transac var hedera = ClientErrors{ NonceTooLow: regexp.MustCompile(`Nonce too low`), NonceTooHigh: regexp.MustCompile(`Nonce too high`), - TerminallyUnderpriced: regexp.MustCompile(`Gas price '(\d+)' is below configured minimum gas price '(\d+)'`), + TerminallyUnderpriced: regexp.MustCompile(`(Gas price '(\d+)' is below configured minimum gas price '(\d+)')|(Gas price too low)`), InsufficientEth: regexp.MustCompile(`Insufficient funds for transfer| failed precheck with status INSUFFICIENT_PAYER_BALANCE`), Fatal: hederaFatal, } From 3462b53b4c21d9b57d74ed36171b366981bd7bcd Mon Sep 17 00:00:00 2001 From: simsonraj Date: Wed, 19 Jun 2024 15:34:15 +0530 Subject: [PATCH 11/14] increased resend interval --- core/chains/evm/config/toml/defaults/Hedera_Testnet.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml b/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml index 7d7ff0ea561..a7f796937da 100644 --- a/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml +++ b/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml @@ -22,9 +22,9 @@ BumpPercent = 20 [Transactions] # to hit throttling you'd need to maintain 15 m gas /sec over a prolonged period of time. -# Because our block times are every 2 secs it's less less likely to happen as compared to other chains -# so setting a gap of 5 blocks to prevent sending transaction during throttling -ResendAfterThreshold = '10s' +# Because Hedera's block times are every 2 secs it's less less likely to happen as compared to other chains +# Setting this to little higher even though Hedera has High TPS, We have seen 10-12s to get the trasaction mined & 20-25s incase of failures +ResendAfterThreshold = '1m' [NodePool] From a8c23c4f41392b79106ee92432146cc90f71a407 Mon Sep 17 00:00:00 2001 From: simsonraj Date: Tue, 2 Jul 2024 17:02:35 +0530 Subject: [PATCH 12/14] testing suggested gas with buffer --- core/chains/evm/gas/suggested_price_estimator.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/core/chains/evm/gas/suggested_price_estimator.go b/core/chains/evm/gas/suggested_price_estimator.go index e947e9109d1..a9f2c087ea5 100644 --- a/core/chains/evm/gas/suggested_price_estimator.go +++ b/core/chains/evm/gas/suggested_price_estimator.go @@ -188,6 +188,9 @@ func (o *SuggestedPriceEstimator) GetLegacyGas(ctx context.Context, _ []byte, Ga } else if err != nil { return } + + gasPrice = o.gasPriceWithBuffer(gasPrice, maxGasPriceWei) + // For L2 chains, submitting a transaction that is not priced high enough will cause the call to fail, so if the cap is lower than the RPC suggested gas price, this transaction cannot succeed if gasPrice != nil && gasPrice.Cmp(maxGasPriceWei) > 0 { return nil, 0, pkgerrors.Errorf("estimated gas price: %s is greater than the maximum gas price configured: %s", gasPrice.String(), maxGasPriceWei.String()) @@ -195,6 +198,17 @@ func (o *SuggestedPriceEstimator) GetLegacyGas(ctx context.Context, _ []byte, Ga return } +func (o *SuggestedPriceEstimator) gasPriceWithBuffer(gasPrice *assets.Wei, maxGasPriceWei *assets.Wei) *assets.Wei { + const BufferPercent = 20 + gasPrice = gasPrice.AddPercentage(BufferPercent) + if gasPrice.Cmp(maxGasPriceWei) > 0 { + o.logger.Warnw("Updated gasPrice with buffer is higher than the max gas price limit. Falling back to max gas price", "gasPriceWithBuffer", gasPrice, "maxGasPriceWei", maxGasPriceWei) + gasPrice = maxGasPriceWei + } + o.logger.Debugw("gasPriceWithBuffer", "updatedGasPrice", gasPrice) + return gasPrice +} + // Refreshes the gas price by making a call to the RPC in case the current one has gone stale. // Adds the larger of BumpPercent and BumpMin configs as a buffer on top of the price returned from the RPC. // The only reason bumping logic would be called on the SuggestedPriceEstimator is if there was a significant price spike From 7bd3aecfd6848d5820c5cc5e4aa30c9d7f32106d Mon Sep 17 00:00:00 2001 From: simsonraj Date: Thu, 4 Jul 2024 23:41:12 +0530 Subject: [PATCH 13/14] test:increasing the timeout to 50s --- core/chains/evm/client/chain_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/chains/evm/client/chain_client.go b/core/chains/evm/client/chain_client.go index 8d1dcb6cc8c..46ecee07d9a 100644 --- a/core/chains/evm/client/chain_client.go +++ b/core/chains/evm/client/chain_client.go @@ -20,7 +20,7 @@ import ( evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" ) -const queryTimeout = 10 * time.Second +const queryTimeout = 50 * time.Second const BALANCE_OF_ADDRESS_FUNCTION_SELECTOR = "0x70a08231" var _ Client = (*chainClient)(nil) From e19656deb6f5e17be19cbe784c6cb4a1a614cfc2 Mon Sep 17 00:00:00 2001 From: Simson Date: Sat, 20 Jul 2024 00:14:59 +0530 Subject: [PATCH 14/14] Update core/chains/evm/client/errors.go Co-authored-by: Jordan Krage --- core/chains/evm/client/errors.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/chains/evm/client/errors.go b/core/chains/evm/client/errors.go index a2d67b939de..24c5157da07 100644 --- a/core/chains/evm/client/errors.go +++ b/core/chains/evm/client/errors.go @@ -71,7 +71,7 @@ func (e *ClientErrors) ErrIs(err error, errorTypes ...int) bool { return false } - cause := pkgerrors.Cause(err).Error() + cause := errors.Unwrap(err).Error() for _, errorType := range errorTypes { if _, ok := (*e)[errorType]; !ok { return false