WIP:Ship hedera

common/client/node_lifecycle.go

@@ -189,7 +189,8 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() {
 			_, num, td := n.StateAndLatest()
 			if outOfSync, liveNodes := n.syncStatus(num, td); outOfSync {
 				// note: there must be another live node for us to be out of sync
-				lggr.Errorw("RPC endpoint has fallen behind", "blockNumber", num, "totalDifficulty", td, "nodeState", n.State())
+				_, highest, greatest := n.nLiveNodes()
+				lggr.Errorw("RPC endpoint has fallen behind", "blockNumber", num, "totalDifficulty", td, "nodeState", n.State(), "highest", highest, "greatest", greatest)
 				if liveNodes < 2 {
 					lggr.Criticalf("RPC endpoint has fallen behind; %s %s", msgCannotDisable, msgDegradedState)
 					continue

core/chains/evm/client/chain_client.go

@@ -20,7 +20,7 @@ import (
 	evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types"
 )
 
-const queryTimeout = 10 * time.Second
+const queryTimeout = 50 * time.Second
 const BALANCE_OF_ADDRESS_FUNCTION_SELECTOR = "0x70a08231"
 
 var _ Client = (*chainClient)(nil)

core/chains/evm/client/errors.go

@@ -70,14 +70,16 @@ func (e *ClientErrors) ErrIs(err error, errorTypes ...int) bool {
 	if err == nil {
 		return false
 	}
+
+	cause := errors.Unwrap(err).Error()
 	for _, errorType := range errorTypes {
 		if _, ok := (*e)[errorType]; !ok {
 			return false
 		}
 		if (*e)[errorType].String() == "" {
 			return false
 		}
-		if (*e)[errorType].MatchString(pkgerrors.Cause(err).Error()) {
+		if (*e)[errorType].MatchString(cause) {
 			return true
 		}
 	}
@@ -249,7 +251,16 @@ var zkEvm = ClientErrors{
 	OutOfCounters: regexp.MustCompile(`(?:: |^)not enough .* counters to continue the execution$`),
 }
 
-var clients = []ClientErrors{parity, geth, arbitrum, metis, substrate, avalanche, nethermind, harmony, besu, erigon, klaytn, celo, zkSync, zkEvm}
+var hederaFatal = regexp.MustCompile(`(: |^)(execution reverted)(:|$) | ^Transaction gas limit '(\d+)' exceeds block gas limit '(\d+)' | ^Transaction gas limit provided '(\d+)' is insufficient of intrinsic gas required '(\d+)' | ^Oversized data:|status INVALID_SIGNATURE`)
+var hedera = ClientErrors{
+	NonceTooLow:           regexp.MustCompile(`Nonce too low`),
+	NonceTooHigh:          regexp.MustCompile(`Nonce too high`),
+	TerminallyUnderpriced: regexp.MustCompile(`(Gas price '(\d+)' is below configured minimum gas price '(\d+)')|(Gas price too low)`),
+	InsufficientEth:       regexp.MustCompile(`Insufficient funds for transfer| failed precheck with status INSUFFICIENT_PAYER_BALANCE`),
+	Fatal:                 hederaFatal,
+}
+
+var clients = []ClientErrors{parity, geth, arbitrum, metis, substrate, avalanche, nethermind, harmony, besu, erigon, klaytn, celo, zkSync, zkEvm, hedera}
 
 // ClientErrorRegexes returns a map of compiled regexes for each error type
 func ClientErrorRegexes(errsRegex config.ClientErrors) *ClientErrors {

core/chains/evm/client/errors_test.go

@@ -44,6 +44,7 @@ func Test_Eth_Errors(t *testing.T) {
 			{"call failed: OldNonce, Current nonce: 22, nonce of rejected tx: 17", true, "Nethermind"},
 			{"nonce too low. allowed nonce range: 427 - 447, actual: 426", true, "zkSync"},
 			{"client error nonce too low", true, "tomlConfig"},
+			{"[Request ID: 2e952947-ffad-408b-aed9-35f3ed152001] Nonce too low. Provided nonce: 15, current nonce: 15", true, "hedera"},
 		}
 
 		for _, test := range tests {
@@ -65,6 +66,7 @@ func Test_Eth_Errors(t *testing.T) {
 			{"nonce too high", true, "Erigon"},
 			{"nonce too high. allowed nonce range: 427 - 477, actual: 527", true, "zkSync"},
 			{"client error nonce too high", true, "tomlConfig"},
+			{"[Request ID: 3ec591b4-9396-49f4-a03f-06c415a7cc6a] Nonce too high. Provided nonce: 16, current nonce: 15", true, "hedera"},
 		}
 
 		for _, test := range tests {
@@ -166,6 +168,7 @@ func Test_Eth_Errors(t *testing.T) {
 			{"max fee per gas less than block base fee", true, "zkSync"},
 			{"virtual machine entered unexpected state. please contact developers and provide transaction details that caused this error. Error description: The operator included transaction with an unacceptable gas price", true, "zkSync"},
 			{"client error terminally underpriced", true, "tomlConfig"},
+			{"[Request ID: e4d09e44-19a4-4eb7-babe-270db4c2ebc9] Gas price '830000000000' is below configured minimum gas price '950000000000'", true, "hedera"},
 		}
 
 		for _, test := range tests {
@@ -214,6 +217,8 @@ func Test_Eth_Errors(t *testing.T) {
 			{"insufficient funds for gas + value. balance: 42719769622667482000, fee: 48098250000000, value: 42719769622667482000", true, "celo"},
 			{"client error insufficient eth", true, "tomlConfig"},
 			{"transaction would cause overdraft", true, "Geth"},
+			{"[Request ID: 9dd78806-58c8-4e6d-89a8-a60962abe705] Error invoking RPC: transaction [email protected] failed precheck with status INSUFFICIENT_PAYER_BALANCE", true, "hedera"},
+			{"[Request ID: 6198d2a3-590f-4724-aae5-69fecead0c49] Insufficient funds for transfer", true, "hedera"},
 		}
 		for _, test := range tests {
 			err = evmclient.NewSendErrorS(test.message)
@@ -380,6 +385,7 @@ func Test_Eth_Errors_Fatal(t *testing.T) {
 		{"Failed to serialize transaction: oversized data. max: 1000000; actual: 1000000", true, "zkSync"},
 
 		{"client error fatal", true, "tomlConfig"},
+		{"[Request ID: d9711488-4c1e-4af2-bc1f-7969913d7b60] Error invoking RPC: transaction [email protected] failed precheck with status INVALID_SIGNATURE", true, "hedera"},
 	}
 
 	for _, test := range tests {

core/chains/evm/config/toml/defaults/Hedera_Mainnet.toml

@@ -0,0 +1,27 @@
+ChainID = '295'
+#considering the 3-5 (6 including a buffer) seconds of finality and 2 seconds block production
+#we set the depth to 6/2 = 3 blocks
+FinalityDepth = 3
+#Hedera has high TPS, so polling less often 
+LogPollInterval = '20s'
+MinIncomingConfirmations = 1
+#turning off the liveness detection, since the Websocket connection is not reliable, can be turned on if needed
+NoNewHeadsThreshold = '0'
+
+[BalanceMonitor]
+Enabled = true
+
+[GasEstimator]
+Mode = 'SuggestedPrice'
+# since Hedera dont have mempool and there's no way for a node to front run or a user to bribe a node to submit the transaction earlier than it's consensus timestamp
+# but they have automated congesting pricing throttling which would mean at high sustained level the gasPrice itself could be increased to prevent malicious behaviour.
+# so wait for 3 blocks before bumping the gas price, the increased gas Price could be adjusted
+BumpThreshold = 3
+BumpMin = '10 gwei'
+BumpPercent = 20
+
+[Transactions]
+# to hit throttling you'd need to maintain 15 m gas /sec over a prolonged period of time.
+# Because our block times are every 2 secs it's less less likely to happen as compared to other chains
+# so setting a gap of 5 blocks to prevent sending transaction during throttling
+ResendAfterThreshold = '10s'

core/chains/evm/config/toml/defaults/Hedera_Testnet.toml

@@ -0,0 +1,32 @@
+ChainID = '296'
+#considering the 3-5 (6 including a buffer) seconds of finality and 2 seconds block production
+#we set the depth to 6/2 = 3 blocks, setting to 10 for safety
+FinalityDepth = 10
+#Hedera has high TPS, so polling less often 
+LogPollInterval = '10s'
+MinIncomingConfirmations = 1
+#turning off the liveness detection, since the Websocket connection is not reliable, can be turned on if needed
+NoNewHeadsThreshold = '0'
+
+[BalanceMonitor]
+Enabled = true
+
+[GasEstimator]
+Mode = 'SuggestedPrice'
+# since Hedera dont have mempool and there's no way for a node to front run or a user to bribe a node to submit the transaction earlier than it's consensus timestamp
+# but they have automated congesting pricing throttling which would mean at high sustained level the gasPrice itself could be increased to prevent malicious behaviour.
+# so wait for 3 blocks before bumping the gas price, the increased gas Price could be adjusted
+BumpThreshold = 3
+BumpMin = '10 gwei'
+BumpPercent = 20
+
+[Transactions]
+# to hit throttling you'd need to maintain 15 m gas /sec over a prolonged period of time.
+# Because Hedera's block times are every 2 secs it's less less likely to happen as compared to other chains
+# Setting this to little higher even though Hedera has High TPS, We have seen 10-12s to get the trasaction mined & 20-25s incase of failures
+ResendAfterThreshold = '1m'
+
+
+[NodePool]
+#Disabling sync threshold check for testing
+SyncThreshold = 0

core/chains/evm/gas/suggested_price_estimator.go

@@ -188,13 +188,27 @@ func (o *SuggestedPriceEstimator) GetLegacyGas(ctx context.Context, _ []byte, Ga
 	} else if err != nil {
 		return
 	}
+
+	gasPrice = o.gasPriceWithBuffer(gasPrice, maxGasPriceWei)
+
 	// For L2 chains, submitting a transaction that is not priced high enough will cause the call to fail, so if the cap is lower than the RPC suggested gas price, this transaction cannot succeed
 	if gasPrice != nil && gasPrice.Cmp(maxGasPriceWei) > 0 {
 		return nil, 0, pkgerrors.Errorf("estimated gas price: %s is greater than the maximum gas price configured: %s", gasPrice.String(), maxGasPriceWei.String())
 	}
 	return
 }
 
+func (o *SuggestedPriceEstimator) gasPriceWithBuffer(gasPrice *assets.Wei, maxGasPriceWei *assets.Wei) *assets.Wei {
+	const BufferPercent = 20
+	gasPrice = gasPrice.AddPercentage(BufferPercent)
+	if gasPrice.Cmp(maxGasPriceWei) > 0 {
+		o.logger.Warnw("Updated gasPrice with buffer is higher than the max gas price limit. Falling back to max gas price", "gasPriceWithBuffer", gasPrice, "maxGasPriceWei", maxGasPriceWei)
+		gasPrice = maxGasPriceWei
+	}
+	o.logger.Debugw("gasPriceWithBuffer", "updatedGasPrice", gasPrice)
+	return gasPrice
+}
+
 // Refreshes the gas price by making a call to the RPC in case the current one has gone stale.
 // Adds the larger of BumpPercent and BumpMin configs as a buffer on top of the price returned from the RPC.
 // The only reason bumping logic would be called on the SuggestedPriceEstimator is if there was a significant price spike

docs/CONFIG.md

@@ -3530,6 +3530,184 @@ GasLimit = 5400000
 
 </p></details>
 
+<details><summary>Hedera Mainnet (295)</summary><p>
+
+```toml
+AutoCreateKey = true
+BlockBackfillDepth = 10
+BlockBackfillSkip = false
+FinalityDepth = 3
+FinalityTagEnabled = false
+LogBackfillBatchSize = 1000
+LogPollInterval = '20s'
+LogKeepBlocksDepth = 100000
+LogPrunePageSize = 0
+BackupLogPollerBlockDelay = 100
+MinIncomingConfirmations = 1
+MinContractPayment = '0.00001 link'
+NonceAutoSync = true
+NoNewHeadsThreshold = '0s'
+RPCDefaultBatchSize = 250
+RPCBlockQueryDelay = 1
+
+[Transactions]
+ForwardersEnabled = false
+MaxInFlight = 16
+MaxQueued = 250
+ReaperInterval = '1h0m0s'
+ReaperThreshold = '168h0m0s'
+ResendAfterThreshold = '10s'
+
+[Transactions.AutoPurge]
+Enabled = false
+
+[BalanceMonitor]
+Enabled = true
+
+[GasEstimator]
+Mode = 'SuggestedPrice'
+PriceDefault = '20 gwei'
+PriceMax = '115792089237316195423570985008687907853269984665.640564039457584007913129639935 tether'
+PriceMin = '1 gwei'
+LimitDefault = 500000
+LimitMax = 500000
+LimitMultiplier = '1'
+LimitTransfer = 21000
+BumpMin = '10 gwei'
+BumpPercent = 20
+BumpThreshold = 3
+EIP1559DynamicFees = false
+FeeCapDefault = '100 gwei'
+TipCapDefault = '1 wei'
+TipCapMin = '1 wei'
+
+[GasEstimator.BlockHistory]
+BatchSize = 25
+BlockHistorySize = 8
+CheckInclusionBlocks = 12
+CheckInclusionPercentile = 90
+TransactionPercentile = 60
+
+[HeadTracker]
+HistoryDepth = 100
+MaxBufferSize = 3
+SamplingInterval = '1s'
+MaxAllowedFinalityDepth = 10000
+FinalityTagBypass = true
+
+[NodePool]
+PollFailureThreshold = 5
+PollInterval = '10s'
+SelectionMode = 'HighestHead'
+SyncThreshold = 5
+LeaseDuration = '0s'
+NodeIsSyncingEnabled = false
+FinalizedBlockPollInterval = '5s'
+
+[OCR]
+ContractConfirmations = 4
+ContractTransmitterTransmitTimeout = '10s'
+DatabaseTimeout = '10s'
+DeltaCOverride = '168h0m0s'
+DeltaCJitterOverride = '1h0m0s'
+ObservationGracePeriod = '1s'
+
+[OCR2]
+[OCR2.Automation]
+GasLimit = 5400000
+```
+
+</p></details>
+
+<details><summary>Hedera Testnet (296)</summary><p>
+
+```toml
+AutoCreateKey = true
+BlockBackfillDepth = 10
+BlockBackfillSkip = false
+FinalityDepth = 3
+FinalityTagEnabled = false
+LogBackfillBatchSize = 1000
+LogPollInterval = '20s'
+LogKeepBlocksDepth = 100000
+LogPrunePageSize = 0
+BackupLogPollerBlockDelay = 100
+MinIncomingConfirmations = 1
+MinContractPayment = '0.00001 link'
+NonceAutoSync = true
+NoNewHeadsThreshold = '0s'
+RPCDefaultBatchSize = 250
+RPCBlockQueryDelay = 1
+
+[Transactions]
+ForwardersEnabled = false
+MaxInFlight = 16
+MaxQueued = 250
+ReaperInterval = '1h0m0s'
+ReaperThreshold = '168h0m0s'
+ResendAfterThreshold = '10s'
+
+[Transactions.AutoPurge]
+Enabled = false
+
+[BalanceMonitor]
+Enabled = true
+
+[GasEstimator]
+Mode = 'SuggestedPrice'
+PriceDefault = '20 gwei'
+PriceMax = '115792089237316195423570985008687907853269984665.640564039457584007913129639935 tether'
+PriceMin = '1 gwei'
+LimitDefault = 500000
+LimitMax = 500000
+LimitMultiplier = '1'
+LimitTransfer = 21000
+BumpMin = '10 gwei'
+BumpPercent = 20
+BumpThreshold = 3
+EIP1559DynamicFees = false
+FeeCapDefault = '100 gwei'
+TipCapDefault = '1 wei'
+TipCapMin = '1 wei'
+
+[GasEstimator.BlockHistory]
+BatchSize = 25
+BlockHistorySize = 8
+CheckInclusionBlocks = 12
+CheckInclusionPercentile = 90
+TransactionPercentile = 60
+
+[HeadTracker]
+HistoryDepth = 100
+MaxBufferSize = 3
+SamplingInterval = '1s'
+MaxAllowedFinalityDepth = 10000
+FinalityTagBypass = true
+
+[NodePool]
+PollFailureThreshold = 5
+PollInterval = '10s'
+SelectionMode = 'HighestHead'
+SyncThreshold = 5
+LeaseDuration = '0s'
+NodeIsSyncingEnabled = false
+FinalizedBlockPollInterval = '5s'
+
+[OCR]
+ContractConfirmations = 4
+ContractTransmitterTransmitTimeout = '10s'
+DatabaseTimeout = '10s'
+DeltaCOverride = '168h0m0s'
+DeltaCJitterOverride = '1h0m0s'
+ObservationGracePeriod = '1s'
+
+[OCR2]
+[OCR2.Automation]
+GasLimit = 5400000
+```
+
+</p></details>
+
 <details><summary>zkSync Sepolia (300)</summary><p>
 
 ```toml