diff --git a/common/client/node_lifecycle.go b/common/client/node_lifecycle.go index 5947774e202..16d77de3a45 100644 --- a/common/client/node_lifecycle.go +++ b/common/client/node_lifecycle.go @@ -189,7 +189,8 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { _, num, td := n.StateAndLatest() if outOfSync, liveNodes := n.syncStatus(num, td); outOfSync { // note: there must be another live node for us to be out of sync - lggr.Errorw("RPC endpoint has fallen behind", "blockNumber", num, "totalDifficulty", td, "nodeState", n.State()) + _, highest, greatest := n.nLiveNodes() + lggr.Errorw("RPC endpoint has fallen behind", "blockNumber", num, "totalDifficulty", td, "nodeState", n.State(), "highest", highest, "greatest", greatest) if liveNodes < 2 { lggr.Criticalf("RPC endpoint has fallen behind; %s %s", msgCannotDisable, msgDegradedState) continue diff --git a/core/chains/evm/client/chain_client.go b/core/chains/evm/client/chain_client.go index 8d1dcb6cc8c..46ecee07d9a 100644 --- a/core/chains/evm/client/chain_client.go +++ b/core/chains/evm/client/chain_client.go @@ -20,7 +20,7 @@ import ( evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" ) -const queryTimeout = 10 * time.Second +const queryTimeout = 50 * time.Second const BALANCE_OF_ADDRESS_FUNCTION_SELECTOR = "0x70a08231" var _ Client = (*chainClient)(nil) diff --git a/core/chains/evm/client/errors.go b/core/chains/evm/client/errors.go index f43b5189c7e..24c5157da07 100644 --- a/core/chains/evm/client/errors.go +++ b/core/chains/evm/client/errors.go @@ -70,6 +70,8 @@ func (e *ClientErrors) ErrIs(err error, errorTypes ...int) bool { if err == nil { return false } + + cause := errors.Unwrap(err).Error() for _, errorType := range errorTypes { if _, ok := (*e)[errorType]; !ok { return false @@ -77,7 +79,7 @@ func (e *ClientErrors) ErrIs(err error, errorTypes ...int) bool { if (*e)[errorType].String() == "" { return false } - if (*e)[errorType].MatchString(pkgerrors.Cause(err).Error()) { + if (*e)[errorType].MatchString(cause) { return true } } @@ -249,7 +251,16 @@ var zkEvm = ClientErrors{ OutOfCounters: regexp.MustCompile(`(?:: |^)not enough .* counters to continue the execution$`), } -var clients = []ClientErrors{parity, geth, arbitrum, metis, substrate, avalanche, nethermind, harmony, besu, erigon, klaytn, celo, zkSync, zkEvm} +var hederaFatal = regexp.MustCompile(`(: |^)(execution reverted)(:|$) | ^Transaction gas limit '(\d+)' exceeds block gas limit '(\d+)' | ^Transaction gas limit provided '(\d+)' is insufficient of intrinsic gas required '(\d+)' | ^Oversized data:|status INVALID_SIGNATURE`) +var hedera = ClientErrors{ + NonceTooLow: regexp.MustCompile(`Nonce too low`), + NonceTooHigh: regexp.MustCompile(`Nonce too high`), + TerminallyUnderpriced: regexp.MustCompile(`(Gas price '(\d+)' is below configured minimum gas price '(\d+)')|(Gas price too low)`), + InsufficientEth: regexp.MustCompile(`Insufficient funds for transfer| failed precheck with status INSUFFICIENT_PAYER_BALANCE`), + Fatal: hederaFatal, +} + +var clients = []ClientErrors{parity, geth, arbitrum, metis, substrate, avalanche, nethermind, harmony, besu, erigon, klaytn, celo, zkSync, zkEvm, hedera} // ClientErrorRegexes returns a map of compiled regexes for each error type func ClientErrorRegexes(errsRegex config.ClientErrors) *ClientErrors { diff --git a/core/chains/evm/client/errors_test.go b/core/chains/evm/client/errors_test.go index cca54c2a4a9..c7a2a590927 100644 --- a/core/chains/evm/client/errors_test.go +++ b/core/chains/evm/client/errors_test.go @@ -44,6 +44,7 @@ func Test_Eth_Errors(t *testing.T) { {"call failed: OldNonce, Current nonce: 22, nonce of rejected tx: 17", true, "Nethermind"}, {"nonce too low. allowed nonce range: 427 - 447, actual: 426", true, "zkSync"}, {"client error nonce too low", true, "tomlConfig"}, + {"[Request ID: 2e952947-ffad-408b-aed9-35f3ed152001] Nonce too low. Provided nonce: 15, current nonce: 15", true, "hedera"}, } for _, test := range tests { @@ -65,6 +66,7 @@ func Test_Eth_Errors(t *testing.T) { {"nonce too high", true, "Erigon"}, {"nonce too high. allowed nonce range: 427 - 477, actual: 527", true, "zkSync"}, {"client error nonce too high", true, "tomlConfig"}, + {"[Request ID: 3ec591b4-9396-49f4-a03f-06c415a7cc6a] Nonce too high. Provided nonce: 16, current nonce: 15", true, "hedera"}, } for _, test := range tests { @@ -166,6 +168,7 @@ func Test_Eth_Errors(t *testing.T) { {"max fee per gas less than block base fee", true, "zkSync"}, {"virtual machine entered unexpected state. please contact developers and provide transaction details that caused this error. Error description: The operator included transaction with an unacceptable gas price", true, "zkSync"}, {"client error terminally underpriced", true, "tomlConfig"}, + {"[Request ID: e4d09e44-19a4-4eb7-babe-270db4c2ebc9] Gas price '830000000000' is below configured minimum gas price '950000000000'", true, "hedera"}, } for _, test := range tests { @@ -214,6 +217,8 @@ func Test_Eth_Errors(t *testing.T) { {"insufficient funds for gas + value. balance: 42719769622667482000, fee: 48098250000000, value: 42719769622667482000", true, "celo"}, {"client error insufficient eth", true, "tomlConfig"}, {"transaction would cause overdraft", true, "Geth"}, + {"[Request ID: 9dd78806-58c8-4e6d-89a8-a60962abe705] Error invoking RPC: transaction 0.0.3041916@1717691931.680570179 failed precheck with status INSUFFICIENT_PAYER_BALANCE", true, "hedera"}, + {"[Request ID: 6198d2a3-590f-4724-aae5-69fecead0c49] Insufficient funds for transfer", true, "hedera"}, } for _, test := range tests { err = evmclient.NewSendErrorS(test.message) @@ -380,6 +385,7 @@ func Test_Eth_Errors_Fatal(t *testing.T) { {"Failed to serialize transaction: oversized data. max: 1000000; actual: 1000000", true, "zkSync"}, {"client error fatal", true, "tomlConfig"}, + {"[Request ID: d9711488-4c1e-4af2-bc1f-7969913d7b60] Error invoking RPC: transaction 0.0.4425573@1718213476.914320044 failed precheck with status INVALID_SIGNATURE", true, "hedera"}, } for _, test := range tests { diff --git a/core/chains/evm/config/toml/defaults/Hedera_Mainnet.toml b/core/chains/evm/config/toml/defaults/Hedera_Mainnet.toml new file mode 100644 index 00000000000..7eac46e69f3 --- /dev/null +++ b/core/chains/evm/config/toml/defaults/Hedera_Mainnet.toml @@ -0,0 +1,27 @@ +ChainID = '295' +#considering the 3-5 (6 including a buffer) seconds of finality and 2 seconds block production +#we set the depth to 6/2 = 3 blocks +FinalityDepth = 3 +#Hedera has high TPS, so polling less often +LogPollInterval = '20s' +MinIncomingConfirmations = 1 +#turning off the liveness detection, since the Websocket connection is not reliable, can be turned on if needed +NoNewHeadsThreshold = '0' + +[BalanceMonitor] +Enabled = true + +[GasEstimator] +Mode = 'SuggestedPrice' +# since Hedera dont have mempool and there's no way for a node to front run or a user to bribe a node to submit the transaction earlier than it's consensus timestamp +# but they have automated congesting pricing throttling which would mean at high sustained level the gasPrice itself could be increased to prevent malicious behaviour. +# so wait for 3 blocks before bumping the gas price, the increased gas Price could be adjusted +BumpThreshold = 3 +BumpMin = '10 gwei' +BumpPercent = 20 + +[Transactions] +# to hit throttling you'd need to maintain 15 m gas /sec over a prolonged period of time. +# Because our block times are every 2 secs it's less less likely to happen as compared to other chains +# so setting a gap of 5 blocks to prevent sending transaction during throttling +ResendAfterThreshold = '10s' diff --git a/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml b/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml new file mode 100644 index 00000000000..a7f796937da --- /dev/null +++ b/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml @@ -0,0 +1,32 @@ +ChainID = '296' +#considering the 3-5 (6 including a buffer) seconds of finality and 2 seconds block production +#we set the depth to 6/2 = 3 blocks, setting to 10 for safety +FinalityDepth = 10 +#Hedera has high TPS, so polling less often +LogPollInterval = '10s' +MinIncomingConfirmations = 1 +#turning off the liveness detection, since the Websocket connection is not reliable, can be turned on if needed +NoNewHeadsThreshold = '0' + +[BalanceMonitor] +Enabled = true + +[GasEstimator] +Mode = 'SuggestedPrice' +# since Hedera dont have mempool and there's no way for a node to front run or a user to bribe a node to submit the transaction earlier than it's consensus timestamp +# but they have automated congesting pricing throttling which would mean at high sustained level the gasPrice itself could be increased to prevent malicious behaviour. +# so wait for 3 blocks before bumping the gas price, the increased gas Price could be adjusted +BumpThreshold = 3 +BumpMin = '10 gwei' +BumpPercent = 20 + +[Transactions] +# to hit throttling you'd need to maintain 15 m gas /sec over a prolonged period of time. +# Because Hedera's block times are every 2 secs it's less less likely to happen as compared to other chains +# Setting this to little higher even though Hedera has High TPS, We have seen 10-12s to get the trasaction mined & 20-25s incase of failures +ResendAfterThreshold = '1m' + + +[NodePool] +#Disabling sync threshold check for testing +SyncThreshold = 0 \ No newline at end of file diff --git a/core/chains/evm/gas/suggested_price_estimator.go b/core/chains/evm/gas/suggested_price_estimator.go index e947e9109d1..a9f2c087ea5 100644 --- a/core/chains/evm/gas/suggested_price_estimator.go +++ b/core/chains/evm/gas/suggested_price_estimator.go @@ -188,6 +188,9 @@ func (o *SuggestedPriceEstimator) GetLegacyGas(ctx context.Context, _ []byte, Ga } else if err != nil { return } + + gasPrice = o.gasPriceWithBuffer(gasPrice, maxGasPriceWei) + // For L2 chains, submitting a transaction that is not priced high enough will cause the call to fail, so if the cap is lower than the RPC suggested gas price, this transaction cannot succeed if gasPrice != nil && gasPrice.Cmp(maxGasPriceWei) > 0 { return nil, 0, pkgerrors.Errorf("estimated gas price: %s is greater than the maximum gas price configured: %s", gasPrice.String(), maxGasPriceWei.String()) @@ -195,6 +198,17 @@ func (o *SuggestedPriceEstimator) GetLegacyGas(ctx context.Context, _ []byte, Ga return } +func (o *SuggestedPriceEstimator) gasPriceWithBuffer(gasPrice *assets.Wei, maxGasPriceWei *assets.Wei) *assets.Wei { + const BufferPercent = 20 + gasPrice = gasPrice.AddPercentage(BufferPercent) + if gasPrice.Cmp(maxGasPriceWei) > 0 { + o.logger.Warnw("Updated gasPrice with buffer is higher than the max gas price limit. Falling back to max gas price", "gasPriceWithBuffer", gasPrice, "maxGasPriceWei", maxGasPriceWei) + gasPrice = maxGasPriceWei + } + o.logger.Debugw("gasPriceWithBuffer", "updatedGasPrice", gasPrice) + return gasPrice +} + // Refreshes the gas price by making a call to the RPC in case the current one has gone stale. // Adds the larger of BumpPercent and BumpMin configs as a buffer on top of the price returned from the RPC. // The only reason bumping logic would be called on the SuggestedPriceEstimator is if there was a significant price spike diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 07a26246d4e..a45a439e8e8 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -3530,6 +3530,184 @@ GasLimit = 5400000

+
Hedera Mainnet (295)

+ +```toml +AutoCreateKey = true +BlockBackfillDepth = 10 +BlockBackfillSkip = false +FinalityDepth = 3 +FinalityTagEnabled = false +LogBackfillBatchSize = 1000 +LogPollInterval = '20s' +LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 +BackupLogPollerBlockDelay = 100 +MinIncomingConfirmations = 1 +MinContractPayment = '0.00001 link' +NonceAutoSync = true +NoNewHeadsThreshold = '0s' +RPCDefaultBatchSize = 250 +RPCBlockQueryDelay = 1 + +[Transactions] +ForwardersEnabled = false +MaxInFlight = 16 +MaxQueued = 250 +ReaperInterval = '1h0m0s' +ReaperThreshold = '168h0m0s' +ResendAfterThreshold = '10s' + +[Transactions.AutoPurge] +Enabled = false + +[BalanceMonitor] +Enabled = true + +[GasEstimator] +Mode = 'SuggestedPrice' +PriceDefault = '20 gwei' +PriceMax = '115792089237316195423570985008687907853269984665.640564039457584007913129639935 tether' +PriceMin = '1 gwei' +LimitDefault = 500000 +LimitMax = 500000 +LimitMultiplier = '1' +LimitTransfer = 21000 +BumpMin = '10 gwei' +BumpPercent = 20 +BumpThreshold = 3 +EIP1559DynamicFees = false +FeeCapDefault = '100 gwei' +TipCapDefault = '1 wei' +TipCapMin = '1 wei' + +[GasEstimator.BlockHistory] +BatchSize = 25 +BlockHistorySize = 8 +CheckInclusionBlocks = 12 +CheckInclusionPercentile = 90 +TransactionPercentile = 60 + +[HeadTracker] +HistoryDepth = 100 +MaxBufferSize = 3 +SamplingInterval = '1s' +MaxAllowedFinalityDepth = 10000 +FinalityTagBypass = true + +[NodePool] +PollFailureThreshold = 5 +PollInterval = '10s' +SelectionMode = 'HighestHead' +SyncThreshold = 5 +LeaseDuration = '0s' +NodeIsSyncingEnabled = false +FinalizedBlockPollInterval = '5s' + +[OCR] +ContractConfirmations = 4 +ContractTransmitterTransmitTimeout = '10s' +DatabaseTimeout = '10s' +DeltaCOverride = '168h0m0s' +DeltaCJitterOverride = '1h0m0s' +ObservationGracePeriod = '1s' + +[OCR2] +[OCR2.Automation] +GasLimit = 5400000 +``` + +

+ +
Hedera Testnet (296)

+ +```toml +AutoCreateKey = true +BlockBackfillDepth = 10 +BlockBackfillSkip = false +FinalityDepth = 3 +FinalityTagEnabled = false +LogBackfillBatchSize = 1000 +LogPollInterval = '20s' +LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 +BackupLogPollerBlockDelay = 100 +MinIncomingConfirmations = 1 +MinContractPayment = '0.00001 link' +NonceAutoSync = true +NoNewHeadsThreshold = '0s' +RPCDefaultBatchSize = 250 +RPCBlockQueryDelay = 1 + +[Transactions] +ForwardersEnabled = false +MaxInFlight = 16 +MaxQueued = 250 +ReaperInterval = '1h0m0s' +ReaperThreshold = '168h0m0s' +ResendAfterThreshold = '10s' + +[Transactions.AutoPurge] +Enabled = false + +[BalanceMonitor] +Enabled = true + +[GasEstimator] +Mode = 'SuggestedPrice' +PriceDefault = '20 gwei' +PriceMax = '115792089237316195423570985008687907853269984665.640564039457584007913129639935 tether' +PriceMin = '1 gwei' +LimitDefault = 500000 +LimitMax = 500000 +LimitMultiplier = '1' +LimitTransfer = 21000 +BumpMin = '10 gwei' +BumpPercent = 20 +BumpThreshold = 3 +EIP1559DynamicFees = false +FeeCapDefault = '100 gwei' +TipCapDefault = '1 wei' +TipCapMin = '1 wei' + +[GasEstimator.BlockHistory] +BatchSize = 25 +BlockHistorySize = 8 +CheckInclusionBlocks = 12 +CheckInclusionPercentile = 90 +TransactionPercentile = 60 + +[HeadTracker] +HistoryDepth = 100 +MaxBufferSize = 3 +SamplingInterval = '1s' +MaxAllowedFinalityDepth = 10000 +FinalityTagBypass = true + +[NodePool] +PollFailureThreshold = 5 +PollInterval = '10s' +SelectionMode = 'HighestHead' +SyncThreshold = 5 +LeaseDuration = '0s' +NodeIsSyncingEnabled = false +FinalizedBlockPollInterval = '5s' + +[OCR] +ContractConfirmations = 4 +ContractTransmitterTransmitTimeout = '10s' +DatabaseTimeout = '10s' +DeltaCOverride = '168h0m0s' +DeltaCJitterOverride = '1h0m0s' +ObservationGracePeriod = '1s' + +[OCR2] +[OCR2.Automation] +GasLimit = 5400000 +``` + +

+
zkSync Sepolia (300)

```toml