diff --git a/lib/rpc/SingleJsonRpcProvider.ts b/lib/rpc/SingleJsonRpcProvider.ts index c208fe5d25..ae086b5f1e 100644 --- a/lib/rpc/SingleJsonRpcProvider.ts +++ b/lib/rpc/SingleJsonRpcProvider.ts @@ -17,7 +17,10 @@ import { Network } from '@ethersproject/networks' import { getProviderId } from './utils' import { ProviderHealthiness } from './ProviderHealthState' -export const MAJOR_METHOD_NAMES: string[] = ['getBlockNumber', 'call', 'send'] +export const GET_BLOCK_NUMBER_METHOD_NAME = 'getBlockNumber' +export const CALL_METHOD_NAME = 'call' +export const SEND_METHOD_NAME = 'send' +export const MAJOR_METHOD_NAMES: string[] = [GET_BLOCK_NUMBER_METHOD_NAME, CALL_METHOD_NAME, SEND_METHOD_NAME] export enum CallType { NORMAL, @@ -180,7 +183,7 @@ export class SingleJsonRpcProvider extends StaticJsonRpcProvider { this.logEvaluateLatency() this.evaluatingLatency = true try { - await (this as any)[`${methodName}_EvaluateLatency`](...args) + return await (this as any)[`${methodName}_EvaluateLatency`](...args) } catch (error: any) { this.log.error({ error }, `Encounter error for shadow evaluate latency call: ${JSON.stringify(error)}`) // Swallow the error. @@ -241,6 +244,22 @@ export class SingleJsonRpcProvider extends StaticJsonRpcProvider { metric.putMetric(`${this.metricPrefix}_send_${method}`, 1, MetricLoggerUnit.Count) } + logRpcResponseMatch(method: string, otherProvider: SingleJsonRpcProvider) { + metric.putMetric( + `${this.metricPrefix}_other_provider_${otherProvider.providerId}_method_${method}_rpc_match`, + 1, + MetricLoggerUnit.Count + ) + } + + logRpcResponseMismatch(method: string, otherProvider: SingleJsonRpcProvider) { + metric.putMetric( + `${this.metricPrefix}_other_provider_${otherProvider.providerId}_method_${method}_rpc_mismatch`, + 1, + MetricLoggerUnit.Count + ) + } + private async wrappedFunctionCall( callType: CallType, fnName: string, diff --git a/lib/rpc/UniJsonRpcProvider.ts b/lib/rpc/UniJsonRpcProvider.ts index 03b36b6804..4bcc0116db 100644 --- a/lib/rpc/UniJsonRpcProvider.ts +++ b/lib/rpc/UniJsonRpcProvider.ts @@ -1,4 +1,10 @@ -import { CallType, MAJOR_METHOD_NAMES, SingleJsonRpcProvider } from './SingleJsonRpcProvider' +import { + CALL_METHOD_NAME, + CallType, + MAJOR_METHOD_NAMES, + SEND_METHOD_NAME, + SingleJsonRpcProvider, +} from './SingleJsonRpcProvider' import { StaticJsonRpcProvider, TransactionRequest } from '@ethersproject/providers' import { isEmpty } from 'lodash' import { ChainId } from '@uniswap/sdk-core' @@ -15,6 +21,8 @@ import { BigNumber, BigNumberish } from '@ethersproject/bignumber' import { Deferrable } from '@ethersproject/properties' import Logger from 'bunyan' import { UniJsonRpcProviderConfig } from './config' +import { EthFeeHistory } from '../util/eth_feeHistory' +import { JsonRpcResponse } from 'hardhat/types' export class UniJsonRpcProvider extends StaticJsonRpcProvider { readonly chainId: ChainId = ChainId.MAINNET @@ -214,7 +222,8 @@ export class UniJsonRpcProvider extends StaticJsonRpcProvider { latency: number, selectedProvider: SingleJsonRpcProvider, methodName: string, - args: any[] + args: any[], + providerResponse: any ): Promise { const healthyProviders = this.providers.filter((provider) => provider.isHealthy()) let count = 0 @@ -230,7 +239,20 @@ export class UniJsonRpcProvider extends StaticJsonRpcProvider { // Within each provider latency shadow evaluation, we should do block I/O, // because NodeJS runs in single thread, so it's important to make sure // we benchmark the latencies correctly based on the single-threaded sequential evaluation. - await provider.evaluateLatency(methodName, args) + const evaluatedProviderResponse = await provider[`evaluateLatency`](methodName, ...args) + // below invocation does not make the call/send RPC return the correct data + // both call and send will return "0x" for some reason + // I have to change to above invocation to make call/send return geniun RPC response + // const evaluatedProviderResponse = await provider.evaluateLatency(methodName, args) + this.compareRpcResponses( + providerResponse, + evaluatedProviderResponse, + selectedProvider, + provider, + methodName, + args + ) + count++ }) ) @@ -242,6 +264,108 @@ export class UniJsonRpcProvider extends StaticJsonRpcProvider { this.log.debug(`Evaluated ${count} other healthy providers`) } + compareRpcResponses( + providerResponse: any, + evaluatedProviderResponse: any, + selectedProvider: SingleJsonRpcProvider, + otherProvider: SingleJsonRpcProvider, + methodName: string, + args: any[] + ) { + switch (methodName) { + case CALL_METHOD_NAME: + // if it's eth_call, then we know the response data type is string, so we can compare directly + if (providerResponse !== evaluatedProviderResponse) { + this.log.error( + { methodName, args }, + `Provider response mismatch: ${providerResponse} from ${selectedProvider.providerId} vs ${evaluatedProviderResponse} from ${otherProvider.providerId}` + ) + selectedProvider.logRpcResponseMismatch(methodName, otherProvider) + } else { + selectedProvider.logRpcResponseMatch(methodName, otherProvider) + } + break + case SEND_METHOD_NAME: + // send is complicated, because it could be eth_call, eth_blockNumber, eth_feeHistory, eth_estimateGas + // so we need to compare the response based on the method name + const underlyingMethodName = args[0] + const stitchedMethodName = `${SEND_METHOD_NAME}_${underlyingMethodName}` + const castedProviderResponse = providerResponse as JsonRpcResponse + const castedEvaluatedProviderResponse = evaluatedProviderResponse as JsonRpcResponse + switch (underlyingMethodName) { + // eth_call result type is string, so we can compare directly + case 'eth_call': + // eth_estimateGas result type is number, so we can compare directly without casting to number type + case 'eth_estimateGas': + if (castedProviderResponse.result !== castedEvaluatedProviderResponse.result) { + this.log.error( + { stitchedMethodName, args }, + `Provider result mismatch: ${castedProviderResponse.result} from ${selectedProvider.providerId} vs ${castedEvaluatedProviderResponse.result} from ${otherProvider.providerId}` + ) + selectedProvider.logRpcResponseMismatch(stitchedMethodName, otherProvider) + } else if (castedProviderResponse.error?.data !== castedEvaluatedProviderResponse.error?.data) { + // when comparing the error, the most important part is the data field + this.log.error( + { stitchedMethodName, args }, + `Provider error mismatch: ${JSON.stringify(castedProviderResponse.error)} from ${ + selectedProvider.providerId + } vs ${JSON.stringify(castedEvaluatedProviderResponse.error)} from ${otherProvider.providerId}` + ) + selectedProvider.logRpcResponseMismatch(stitchedMethodName, otherProvider) + } else { + selectedProvider.logRpcResponseMatch(stitchedMethodName, otherProvider) + } + break + case 'eth_feeHistory': + if (castedProviderResponse.result && castedEvaluatedProviderResponse.result) { + const ethFeeHistory = castedProviderResponse.result as EthFeeHistory + const evaluatedEthFeeHistory = castedEvaluatedProviderResponse.result as EthFeeHistory + + const mismatch = + ethFeeHistory.oldestBlock !== evaluatedEthFeeHistory.oldestBlock || + JSON.stringify(ethFeeHistory.reward) !== JSON.stringify(evaluatedEthFeeHistory.reward) || + JSON.stringify(ethFeeHistory.baseFeePerGas) !== JSON.stringify(evaluatedEthFeeHistory.baseFeePerGas) || + JSON.stringify(ethFeeHistory.gasUsedRatio) !== JSON.stringify(evaluatedEthFeeHistory.gasUsedRatio) || + JSON.stringify(ethFeeHistory.baseFeePerBlobGas) !== + JSON.stringify(evaluatedEthFeeHistory.baseFeePerBlobGas) || + JSON.stringify(ethFeeHistory.blobGasUsedRatio) !== + JSON.stringify(evaluatedEthFeeHistory.blobGasUsedRatio) + if (mismatch) { + this.log.error( + { stitchedMethodName, args }, + `Provider result mismatch: ${JSON.stringify(ethFeeHistory)} from ${ + selectedProvider.providerId + } vs ${JSON.stringify(evaluatedEthFeeHistory)} from ${otherProvider.providerId}` + ) + selectedProvider.logRpcResponseMismatch(stitchedMethodName, otherProvider) + } else { + selectedProvider.logRpcResponseMatch(stitchedMethodName, otherProvider) + } + } else if (castedProviderResponse.error?.data !== castedEvaluatedProviderResponse.error?.data) { + // when comparing the error, the most important part is the data field + this.log.error( + { stitchedMethodName, args }, + `Provider error mismatch: ${JSON.stringify(castedProviderResponse.error)} from ${ + selectedProvider.providerId + } vs ${JSON.stringify(castedEvaluatedProviderResponse.error)} from ${otherProvider.providerId}` + ) + selectedProvider.logRpcResponseMismatch(stitchedMethodName, otherProvider) + } + + break + default: + // if it's get block number, there's no guarantee that two providers will return the same block number + // since the node might be syncing, so we don't need to compare the response + return + } + break + default: + // if it's get block number, there's no guarantee that two providers will return the same block number + // since the node might be syncing, so we don't need to compare the response + return + } + } + logProviderHealthiness() { for (const provider of this.providers.filter((provider) => provider.isHealthy())) { this.log.debug(`Healthy provider: ${provider.url}`) @@ -289,9 +413,10 @@ export class UniJsonRpcProvider extends StaticJsonRpcProvider { const selectedProvider = this.selectPreferredProvider(sessionId) selectedProvider.logProviderSelection() let latency = 0 + let result try { const start = Date.now() - const result = await (selectedProvider as any)[`${fnName}`](...args) + result = await (selectedProvider as any)[`${fnName}`](...args) latency = Date.now() - start return result } catch (error: any) { @@ -308,7 +433,7 @@ export class UniJsonRpcProvider extends StaticJsonRpcProvider { sessionId ) { // fire and forget to evaluate latency of other healthy providers - this.checkOtherHealthyProvider(latency, selectedProvider, fnName, args) + this.checkOtherHealthyProvider(latency, selectedProvider, fnName, args, result) } if (Math.random() < this.healthCheckSampleProb && sessionId) { diff --git a/lib/util/eth_feeHistory.ts b/lib/util/eth_feeHistory.ts new file mode 100644 index 0000000000..bae458ced8 --- /dev/null +++ b/lib/util/eth_feeHistory.ts @@ -0,0 +1,8 @@ +export type EthFeeHistory = { + oldestBlock: string + reward: string[] + baseFeePerGas: string[] + gasUsedRatio: number[] + baseFeePerBlobGas: string[] + blobGasUsedRatio: number[] +} diff --git a/test/mocha/unit/rpc/UniJsonRpcProvider.test.ts b/test/mocha/unit/rpc/UniJsonRpcProvider.test.ts index 5974cb0791..4d61316aff 100644 --- a/test/mocha/unit/rpc/UniJsonRpcProvider.test.ts +++ b/test/mocha/unit/rpc/UniJsonRpcProvider.test.ts @@ -11,6 +11,8 @@ import { import { SingleJsonRpcProvider } from '../../../../lib/rpc/SingleJsonRpcProvider' import { default as bunyan } from 'bunyan' import { ProviderHealthiness } from '../../../../lib/rpc/ProviderHealthState' +import { JsonRpcResponse } from 'hardhat/types' +import { EthFeeHistory } from '../../../../lib/util/eth_feeHistory' const UNI_PROVIDER_TEST_CONFIG: UniJsonRpcProviderConfig = { HEALTH_EVALUATION_WAIT_PERIOD_IN_S: 0, @@ -1007,4 +1009,156 @@ describe('UniJsonRpcProvider', () => { expect(spy1.callCount).to.equal(1) expect(spy2.callCount).to.equal(1) }) + + it('Test compare RPC result for eth_call with same results', async () => { + const rpcProviders = createNewSingleJsonRpcProviders() + const selectedProvider = rpcProviders[0] + const otherProvider = rpcProviders[1] + const spy = sandbox.spy(selectedProvider, 'logRpcResponseMatch') + + uniProvider = new UniJsonRpcProvider(ChainId.MAINNET, rpcProviders, log, UNI_PROVIDER_TEST_CONFIG, 1.0, 1) + + uniProvider.compareRpcResponses('0x123', '0x123', selectedProvider, otherProvider, 'call', []) + + expect(spy.callCount).to.equal(1) + }) + + it('Test compare RPC result for eth_call with different results', async () => { + const rpcProviders = createNewSingleJsonRpcProviders() + const selectedProvider = rpcProviders[0] + const otherProvider = rpcProviders[1] + const spy = sandbox.spy(selectedProvider, 'logRpcResponseMismatch') + + uniProvider = new UniJsonRpcProvider(ChainId.MAINNET, rpcProviders, log, UNI_PROVIDER_TEST_CONFIG, 1.0, 1) + + uniProvider.compareRpcResponses('0x321', '0x123', selectedProvider, otherProvider, 'call', []) + + expect(spy.callCount).to.equal(1) + }) + + it('Test compare RPC result for eth_estimateGas with same results', async () => { + const rpcProviders = createNewSingleJsonRpcProviders() + const selectedProvider = rpcProviders[0] + const otherProvider = rpcProviders[1] + const spy = sandbox.spy(selectedProvider, 'logRpcResponseMatch') + + uniProvider = new UniJsonRpcProvider(ChainId.MAINNET, rpcProviders, log, UNI_PROVIDER_TEST_CONFIG, 1.0, 1) + + const providerResult: JsonRpcResponse = { jsonrpc: '2.0', result: '0x123', id: 76 } + const otherProviderResult: JsonRpcResponse = { jsonrpc: '2.0', result: '0x123', id: 76 } + uniProvider.compareRpcResponses(providerResult, otherProviderResult, selectedProvider, otherProvider, 'send', [ + 'eth_call', + ]) + + expect(spy.callCount).to.equal(1) + }) + + it('Test compare RPC result for eth_estimateGas with different results', async () => { + const rpcProviders = createNewSingleJsonRpcProviders() + const selectedProvider = rpcProviders[0] + const otherProvider = rpcProviders[1] + const spy = sandbox.spy(selectedProvider, 'logRpcResponseMismatch') + + uniProvider = new UniJsonRpcProvider(ChainId.MAINNET, rpcProviders, log, UNI_PROVIDER_TEST_CONFIG, 1.0, 1) + + const providerResult: JsonRpcResponse = { jsonrpc: '2.0', result: '0x123', id: 76 } + const otherProviderResult: JsonRpcResponse = { jsonrpc: '2.0', result: '0x321', id: 76 } + uniProvider.compareRpcResponses(providerResult, otherProviderResult, selectedProvider, otherProvider, 'send', [ + 'eth_call', + ]) + + expect(spy.callCount).to.equal(1) + }) + + it('Test compare RPC error for eth_estimateGas with same errors', async () => { + const rpcProviders = createNewSingleJsonRpcProviders() + const selectedProvider = rpcProviders[0] + const otherProvider = rpcProviders[1] + const spy = sandbox.spy(selectedProvider, 'logRpcResponseMatch') + + uniProvider = new UniJsonRpcProvider(ChainId.MAINNET, rpcProviders, log, UNI_PROVIDER_TEST_CONFIG, 1.0, 1) + + const providerError = { code: '123', data: '0x123', error: 'CALL_EXCEPTION' } + const otherProviderError = { code: '123', data: '0x123', error: 'CALL_EXCEPTION' } + uniProvider.compareRpcResponses(providerError, otherProviderError, selectedProvider, otherProvider, 'send', [ + 'eth_call', + ]) + + expect(spy.callCount).to.equal(1) + }) + + it('Test compare RPC error for eth_estimateGas with different errors', async () => { + const rpcProviders = createNewSingleJsonRpcProviders() + const selectedProvider = rpcProviders[0] + const otherProvider = rpcProviders[1] + const spy = sandbox.spy(selectedProvider, 'logRpcResponseMatch') + + uniProvider = new UniJsonRpcProvider(ChainId.MAINNET, rpcProviders, log, UNI_PROVIDER_TEST_CONFIG, 1.0, 1) + + const providerError = { code: '123', data: '0x321', error: 'CALL_EXCEPTION' } + const otherProviderError = { code: '123', data: '0x123', error: 'CALL_EXCEPTION' } + uniProvider.compareRpcResponses(providerError, otherProviderError, selectedProvider, otherProvider, 'send', [ + 'eth_call', + ]) + + expect(spy.callCount).to.equal(1) + }) + + it('Test compare RPC result for eth_feeHistory with different results, but one is a number and the other is a string', async () => { + const rpcProviders = createNewSingleJsonRpcProviders() + const selectedProvider = rpcProviders[0] + const otherProvider = rpcProviders[1] + const spy = sandbox.spy(selectedProvider, 'logRpcResponseMatch') + + uniProvider = new UniJsonRpcProvider(ChainId.MAINNET, rpcProviders, log, UNI_PROVIDER_TEST_CONFIG, 1.0, 1) + + const ethFeeHistory: EthFeeHistory = { + oldestBlock: '0x1347665', + reward: ['0x21f43815'], + baseFeePerGas: ['0x7750ad57'], + gasUsedRatio: [0.4496709], + baseFeePerBlobGas: ['0x1'], + blobGasUsedRatio: [0.4496709], + } + const providerResult: JsonRpcResponse = { jsonrpc: '2.0', result: ethFeeHistory, id: 76 } + const otherProviderResult: JsonRpcResponse = { jsonrpc: '2.0', result: ethFeeHistory, id: 76 } + uniProvider.compareRpcResponses(providerResult, otherProviderResult, selectedProvider, otherProvider, 'send', [ + 'eth_feeHistory', + ]) + + expect(spy.callCount).to.equal(1) + }) + + it('Test compare RPC result for eth_feeHistory with different results', async () => { + const rpcProviders = createNewSingleJsonRpcProviders() + const selectedProvider = rpcProviders[0] + const otherProvider = rpcProviders[1] + const spy = sandbox.spy(selectedProvider, 'logRpcResponseMismatch') + + uniProvider = new UniJsonRpcProvider(ChainId.MAINNET, rpcProviders, log, UNI_PROVIDER_TEST_CONFIG, 1.0, 1) + + const ethFeeHistory: EthFeeHistory = { + oldestBlock: '0x1347665', + reward: ['0x21f43815'], + baseFeePerGas: ['0x7750ad57'], + gasUsedRatio: [0.4496709], + baseFeePerBlobGas: ['0x1'], + blobGasUsedRatio: [0.4496709], + } + const ethFeeHistory2: EthFeeHistory = { + oldestBlock: '0x1347661', + reward: ['0x21f43815'], + baseFeePerGas: ['0x7750ad57'], + gasUsedRatio: [0.4496709], + baseFeePerBlobGas: ['0x1'], + blobGasUsedRatio: [0.4496709], + } + const providerResult: JsonRpcResponse = { jsonrpc: '2.0', result: ethFeeHistory, id: 76 } + const otherProviderResult: JsonRpcResponse = { jsonrpc: '2.0', result: ethFeeHistory2, id: 76 } + uniProvider.compareRpcResponses(providerResult, otherProviderResult, selectedProvider, otherProvider, 'send', [ + 'eth_feeHistory', + ]) + + expect(spy.callCount).to.equal(1) + }) })