From 2b3db572fc352a908cf8b67569bb9a4198b3e601 Mon Sep 17 00:00:00 2001 From: Adrian Adamiak Date: Fri, 29 Sep 2023 16:31:23 +0200 Subject: [PATCH] Fetch logs in batches, with cache-friendly ranges (#43) * Fetch logs in batches, with cache-friendly ranges * Configure GETLOGS_MAX_RANGE for each project in env * Simplify getLogs implementation * Rename variable * Fix test * A tiny refactor * Tiny fix (let->const) * Add comment * Add changeset file --- .changeset/wild-avocados-push.md | 5 + .../discovery/src/config/config.discovery.ts | 39 ++++- .../src/discovery/DiscoveryLogger.ts | 5 + .../analysis/AddressAnalyzer.test.ts | 12 +- .../src/discovery/analysis/AddressAnalyzer.ts | 8 +- .../output/toDiscoveryOutput.test.ts | 1 + .../provider/DiscoveryProvider.test.ts | 152 ++++++++++++++++++ .../discovery/provider/DiscoveryProvider.ts | 59 ++++++- .../discovery/provider/ProviderWithCache.ts | 20 ++- .../discovery/src/discovery/runDiscovery.ts | 15 +- packages/discovery/src/singleDiscovery.ts | 1 + 11 files changed, 301 insertions(+), 16 deletions(-) create mode 100644 .changeset/wild-avocados-push.md create mode 100644 packages/discovery/src/discovery/provider/DiscoveryProvider.test.ts diff --git a/.changeset/wild-avocados-push.md b/.changeset/wild-avocados-push.md new file mode 100644 index 00000000..d7eb7f5e --- /dev/null +++ b/.changeset/wild-avocados-push.md @@ -0,0 +1,5 @@ +--- +'@l2beat/discovery': minor +--- + +Support fetching logs in batches diff --git a/packages/discovery/src/config/config.discovery.ts b/packages/discovery/src/config/config.discovery.ts index fe5f7ba7..ce544c02 100644 --- a/packages/discovery/src/config/config.discovery.ts +++ b/packages/discovery/src/config/config.discovery.ts @@ -20,6 +20,7 @@ export function getDiscoveryCliConfig(cli: CliParameters): DiscoveryCliConfig { const discoveryEnabled = cli.mode === 'discover' const singleDiscoveryEnabled = cli.mode === 'single-discovery' const invertEnabled = cli.mode === 'invert' + const chain = getChainConfig(cli.chain) return { invert: invertEnabled && { @@ -33,12 +34,13 @@ export function getDiscoveryCliConfig(cli: CliParameters): DiscoveryCliConfig { dryRun: cli.dryRun, dev: cli.dev, blockNumber: env.optionalInteger('DISCOVERY_BLOCK_NUMBER'), + getLogsMaxRange: chain.rpcGetLogsMaxRange, }, singleDiscovery: singleDiscoveryEnabled && { address: cli.address, chainId: cli.chain, }, - chain: getChainConfig(cli.chain), + chain, } } @@ -50,6 +52,9 @@ function getChainConfig(chainId: ChainId): DiscoveryChainConfig { return { chainId: ChainId.ETHEREUM, rpcUrl: env.string('DISCOVERY_ETHEREUM_RPC_URL'), + rpcGetLogsMaxRange: env.optionalInteger( + 'DISCOVERY_ETHEREUM_RPC_GETLOGS_MAX_RANGE', + ), etherscanApiKey: env.string('DISCOVERY_ETHEREUM_ETHERSCAN_API_KEY'), etherscanUrl: 'https://api.etherscan.io/api', } @@ -57,6 +62,9 @@ function getChainConfig(chainId: ChainId): DiscoveryChainConfig { return { chainId: ChainId.ARBITRUM, rpcUrl: env.string('DISCOVERY_ARBITRUM_RPC_URL'), + rpcGetLogsMaxRange: env.optionalInteger( + 'DISCOVERY_ARBITRUM_RPC_GETLOGS_MAX_RANGE', + ), etherscanApiKey: env.string('DISCOVERY_ARBITRUM_ETHERSCAN_API_KEY'), etherscanUrl: 'https://api.arbiscan.io/api', } @@ -64,6 +72,9 @@ function getChainConfig(chainId: ChainId): DiscoveryChainConfig { return { chainId: ChainId.OPTIMISM, rpcUrl: env.string('DISCOVERY_OPTIMISM_RPC_URL'), + rpcGetLogsMaxRange: env.optionalInteger( + 'DISCOVERY_OPTIMISM_RPC_GETLOGS_MAX_RANGE', + ), etherscanApiKey: env.string('DISCOVERY_OPTIMISM_ETHERSCAN_API_KEY'), etherscanUrl: 'https://api-optimistic.etherscan.io/api', } @@ -71,6 +82,9 @@ function getChainConfig(chainId: ChainId): DiscoveryChainConfig { return { chainId: ChainId.POLYGON_POS, rpcUrl: env.string('DISCOVERY_POLYGON_POS_RPC_URL'), + rpcGetLogsMaxRange: env.optionalInteger( + 'DISCOVERY_POLYGON_POS_RPC_GETLOGS_MAX_RANGE', + ), etherscanApiKey: env.string('DISCOVERY_POLYGON_POS_ETHERSCAN_API_KEY'), etherscanUrl: 'https://api.polygonscan.com/api', } @@ -78,6 +92,9 @@ function getChainConfig(chainId: ChainId): DiscoveryChainConfig { return { chainId: ChainId.BSC, rpcUrl: env.string('DISCOVERY_BSC_RPC_URL'), + rpcGetLogsMaxRange: env.optionalInteger( + 'DISCOVERY_BSC_RPC_GETLOGS_MAX_RANGE', + ), etherscanApiKey: env.string('DISCOVERY_BSC_ETHERSCAN_API_KEY'), etherscanUrl: 'https://api.bscscan.com/api', } @@ -85,6 +102,9 @@ function getChainConfig(chainId: ChainId): DiscoveryChainConfig { return { chainId: ChainId.AVALANCHE, rpcUrl: env.string('DISCOVERY_AVALANCHE_RPC_URL'), + rpcGetLogsMaxRange: env.optionalInteger( + 'DISCOVERY_AVALANCHE_RPC_GETLOGS_MAX_RANGE', + ), etherscanApiKey: env.string('DISCOVERY_AVALANCHE_ETHERSCAN_API_KEY'), etherscanUrl: 'https://api.snowtrace.io/api', } @@ -92,6 +112,9 @@ function getChainConfig(chainId: ChainId): DiscoveryChainConfig { return { chainId: ChainId.CELO, rpcUrl: env.string('DISCOVERY_CELO_RPC_URL'), + rpcGetLogsMaxRange: env.optionalInteger( + 'DISCOVERY_CELO_RPC_GETLOGS_MAX_RANGE', + ), etherscanApiKey: env.string('DISCOVERY_CELO_ETHERSCAN_API_KEY'), etherscanUrl: 'https://api.celoscan.io/api', } @@ -99,6 +122,9 @@ function getChainConfig(chainId: ChainId): DiscoveryChainConfig { return { chainId: ChainId.LINEA, rpcUrl: env.string('DISCOVERY_LINEA_RPC_URL'), + rpcGetLogsMaxRange: env.optionalInteger( + 'DISCOVERY_LINEA_RPC_GETLOGS_MAX_RANGE', + ), etherscanApiKey: env.string('DISCOVERY_LINEA_ETHERSCAN_API_KEY'), etherscanUrl: 'https://api.lineascan.build/api', } @@ -106,6 +132,9 @@ function getChainConfig(chainId: ChainId): DiscoveryChainConfig { return { chainId: ChainId.BASE, rpcUrl: env.string('DISCOVERY_BASE_RPC_URL'), + rpcGetLogsMaxRange: env.optionalInteger( + 'DISCOVERY_BASE_RPC_GETLOGS_MAX_RANGE', + ), etherscanApiKey: env.string('DISCOVERY_BASE_ETHERSCAN_API_KEY'), etherscanUrl: 'https://api.basescan.org/api', } @@ -113,6 +142,9 @@ function getChainConfig(chainId: ChainId): DiscoveryChainConfig { return { chainId: ChainId.POLYGON_ZKEVM, rpcUrl: env.string('DISCOVERY_POLYGON_ZKEVM_RPC_URL'), + rpcGetLogsMaxRange: env.optionalInteger( + 'DISCOVERY_POLYGON_ZKEVM_RPC_GETLOGS_MAX_RANGE', + ), etherscanApiKey: env.string( 'DISCOVERY_POLYGON_ZKEVM_ETHERSCAN_API_KEY', ), @@ -122,6 +154,9 @@ function getChainConfig(chainId: ChainId): DiscoveryChainConfig { return { chainId: ChainId.GNOSIS, rpcUrl: env.string('DISCOVERY_GNOSIS_RPC_URL'), + rpcGetLogsMaxRange: env.optionalInteger( + 'DISCOVERY_GNOSIS_RPC_GETLOGS_MAX_RANGE', + ), etherscanApiKey: env.string('DISCOVERY_GNOSIS_ETHERSCAN_API_KEY'), etherscanUrl: 'https://api.gnosisscan.io/api', } @@ -145,6 +180,7 @@ export interface DiscoveryModuleConfig { readonly dryRun?: boolean readonly dev?: boolean readonly blockNumber?: number + readonly getLogsMaxRange?: number } export interface SingleDiscoveryModuleConfig { @@ -155,6 +191,7 @@ export interface SingleDiscoveryModuleConfig { export interface DiscoveryChainConfig { chainId: ChainId rpcUrl: string + rpcGetLogsMaxRange?: number etherscanApiKey: string etherscanUrl: string } diff --git a/packages/discovery/src/discovery/DiscoveryLogger.ts b/packages/discovery/src/discovery/DiscoveryLogger.ts index 13694f5c..3b5c0a9c 100644 --- a/packages/discovery/src/discovery/DiscoveryLogger.ts +++ b/packages/discovery/src/discovery/DiscoveryLogger.ts @@ -103,6 +103,11 @@ export class DiscoveryLogger { )}`, ) } + + logFetchingEvents(fromBlock: number, toBlock: number): void { + const text = `Fetching events in range ${fromBlock} - ${toBlock}` + this.log(` ${chalk.gray(text)}`) + } } function dots(length: number): string { diff --git a/packages/discovery/src/discovery/analysis/AddressAnalyzer.test.ts b/packages/discovery/src/discovery/analysis/AddressAnalyzer.test.ts index 03baac41..88ec2ae1 100644 --- a/packages/discovery/src/discovery/analysis/AddressAnalyzer.test.ts +++ b/packages/discovery/src/discovery/analysis/AddressAnalyzer.test.ts @@ -60,7 +60,10 @@ describe(AddressAnalyzer.name, () => { const addressAnalyzer = new AddressAnalyzer( mockObject({ getCode: async () => Bytes.fromHex('0x1234'), - getDeploymentTimestamp: async () => new UnixTime(1234), + getDeploymentInfo: async () => ({ + timestamp: new UnixTime(1234), + blockNumber: 9876, + }), }), mockObject({ detectProxy: async () => ({ @@ -100,6 +103,7 @@ describe(AddressAnalyzer.name, () => { derivedName: undefined, isVerified: true, deploymentTimestamp: new UnixTime(1234), + deploymentBlockNumber: 9876, upgradeability: { type: 'EIP1967 proxy', implementation, admin }, implementations: [implementation], values: { owner: owner.toString() }, @@ -130,7 +134,10 @@ describe(AddressAnalyzer.name, () => { const addressAnalyzer = new AddressAnalyzer( mockObject({ getCode: async () => Bytes.fromHex('0x1234'), - getDeploymentTimestamp: async () => new UnixTime(1234), + getDeploymentInfo: async () => ({ + timestamp: new UnixTime(1234), + blockNumber: 9876, + }), }), mockObject({ detectProxy: async () => ({ @@ -170,6 +177,7 @@ describe(AddressAnalyzer.name, () => { address, isVerified: false, deploymentTimestamp: new UnixTime(1234), + deploymentBlockNumber: 9876, upgradeability: { type: 'EIP1967 proxy', implementation, admin }, implementations: [implementation], values: { owner: owner.toString() }, diff --git a/packages/discovery/src/discovery/analysis/AddressAnalyzer.ts b/packages/discovery/src/discovery/analysis/AddressAnalyzer.ts index 12563cdd..ea90de17 100644 --- a/packages/discovery/src/discovery/analysis/AddressAnalyzer.ts +++ b/packages/discovery/src/discovery/analysis/AddressAnalyzer.ts @@ -20,6 +20,7 @@ export interface AnalyzedContract { address: EthereumAddress name: string deploymentTimestamp: UnixTime + deploymentBlockNumber: number derivedName: string | undefined isVerified: boolean upgradeability: UpgradeabilityParameters @@ -55,8 +56,10 @@ export class AddressAnalyzer { return { analysis: { type: 'EOA', address }, relatives: [] } } - const deploymentTimestamp = - await this.provider.getDeploymentTimestamp(address) + const { + timestamp: deploymentTimestamp, + blockNumber: deploymentBlockNumber, + } = await this.provider.getDeploymentInfo(address) const proxy = await this.proxyDetector.detectProxy( address, @@ -86,6 +89,7 @@ export class AddressAnalyzer { isVerified: sources.isVerified, address, deploymentTimestamp, + deploymentBlockNumber, upgradeability: proxy?.upgradeability ?? { type: 'immutable' }, implementations: proxy?.implementations ?? [], values: values ?? {}, diff --git a/packages/discovery/src/discovery/output/toDiscoveryOutput.test.ts b/packages/discovery/src/discovery/output/toDiscoveryOutput.test.ts index e83a68af..1fa9157a 100644 --- a/packages/discovery/src/discovery/output/toDiscoveryOutput.test.ts +++ b/packages/discovery/src/discovery/output/toDiscoveryOutput.test.ts @@ -14,6 +14,7 @@ describe(processAnalysis.name, () => { values: {}, isVerified: true, deploymentTimestamp: new UnixTime(1234), + deploymentBlockNumber: 9876, upgradeability: { type: 'immutable' } as UpgradeabilityParameters, implementations: [], abis: {}, diff --git a/packages/discovery/src/discovery/provider/DiscoveryProvider.test.ts b/packages/discovery/src/discovery/provider/DiscoveryProvider.test.ts new file mode 100644 index 00000000..148729a7 --- /dev/null +++ b/packages/discovery/src/discovery/provider/DiscoveryProvider.test.ts @@ -0,0 +1,152 @@ +import { expect, mockFn, MockObject, mockObject } from 'earl' +import { providers } from 'ethers' + +import { EthereumAddress } from '../../utils/EthereumAddress' +import { EtherscanLikeClient } from '../../utils/EtherscanLikeClient' +import { DiscoveryLogger } from '../DiscoveryLogger' +import { DiscoveryProvider } from './DiscoveryProvider' + +const rangesFromCalls = (provider: MockObject) => + provider.getLogs.calls.map((call) => [ + call.args[0].fromBlock, + call.args[0].toBlock, + ]) + +const GETLOGS_MAX_RANGE = 10000 + +describe(DiscoveryProvider.name, () => { + describe(DiscoveryProvider.prototype.getLogs.name, () => { + const etherscanLikeClientMock = mockObject({}) + const address = EthereumAddress.random() + const topics = ['testTopic'] + let providerMock: MockObject + let discoveryProviderMock: DiscoveryProvider + + beforeEach(() => { + providerMock = mockObject({ + getLogs: mockFn().resolvesTo([]), + }) + discoveryProviderMock = new DiscoveryProvider( + providerMock, + etherscanLikeClientMock, + DiscoveryLogger.SILENT, + GETLOGS_MAX_RANGE, + ) + discoveryProviderMock.getDeploymentInfo = mockFn().resolvesTo({ + blockNumber: 0, + timestamp: 0, + }) + }) + + it('handles simple range', async () => { + await discoveryProviderMock.getLogs(address, topics, 5000, 35000) + const ranges = rangesFromCalls(providerMock) + expect(ranges).toEqual([ + [5000, 9999], + [10000, 19999], + [20000, 29999], + [30000, 35000], + ]) + }) + + it('handles range at boundaries', async () => { + await discoveryProviderMock.getLogs(address, topics, 10000, 39999) + const ranges = rangesFromCalls(providerMock) + expect(ranges).toEqual([ + [10000, 19999], + [20000, 29999], + [30000, 39999], + ]) + }) + + it('handles range at +/- 1 of boundaries', async () => { + await discoveryProviderMock.getLogs(address, topics, 9999, 30000) + const ranges = rangesFromCalls(providerMock) + expect(ranges).toEqual([ + [9999, 9999], + [10000, 19999], + [20000, 29999], + [30000, 30000], + ]) + }) + + it('handles range where from and to are equal and multiply or range', async () => { + await discoveryProviderMock.getLogs(address, topics, 10000, 10000) + const ranges = rangesFromCalls(providerMock) + expect(ranges).toEqual([[10000, 10000]]) + }) + + it('handles range where from and to are -1 of multiply or range', async () => { + await discoveryProviderMock.getLogs(address, topics, 9999, 9999) + const ranges = rangesFromCalls(providerMock) + expect(ranges).toEqual([[9999, 9999]]) + }) + + it('handles range where from and to are +1 of multiply or range', async () => { + await discoveryProviderMock.getLogs(address, topics, 10001, 10001) + const ranges = rangesFromCalls(providerMock) + expect(ranges).toEqual([[10001, 10001]]) + }) + + it('handles range [0,0]', async () => { + await discoveryProviderMock.getLogs(address, topics, 0, 0) + const ranges = rangesFromCalls(providerMock) + expect(ranges).toEqual([[0, 0]]) + }) + + it('handles range [1,1]', async () => { + await discoveryProviderMock.getLogs(address, topics, 1, 1) + const ranges = rangesFromCalls(providerMock) + expect(ranges).toEqual([[1, 1]]) + }) + + it('handles range inside boundaries', async () => { + await discoveryProviderMock.getLogs(address, topics, 1400, 1600) + const ranges = rangesFromCalls(providerMock) + expect(ranges).toEqual([[1400, 1600]]) + }) + + it('handles getLogsMaxRange undefined (no range)', async () => { + providerMock = mockObject({ + getLogs: mockFn().resolvesTo([]), + }) + discoveryProviderMock = new DiscoveryProvider( + providerMock, + etherscanLikeClientMock, + DiscoveryLogger.SILENT, + undefined, // PROVIDING UNDEFINED for getLogsMaxRange, so no batching + ) + discoveryProviderMock.getDeploymentInfo = mockFn().resolvesTo({ + blockNumber: 0, + timestamp: 0, + }) + await discoveryProviderMock.getLogs(address, topics, 5000, 35000) + const ranges = rangesFromCalls(providerMock) + expect(ranges).toEqual([[5000, 35000]]) + }) + + it('starts with deployment block if bigger then fromBlock', async () => { + providerMock = mockObject({ + getLogs: mockFn().resolvesTo([]), + }) + discoveryProviderMock = new DiscoveryProvider( + providerMock, + etherscanLikeClientMock, + DiscoveryLogger.SILENT, + GETLOGS_MAX_RANGE, + ) + discoveryProviderMock.getDeploymentInfo = mockFn().resolvesTo({ + blockNumber: 16000, + timestamp: 0, + }) + + await discoveryProviderMock.getLogs(address, topics, 5000, 35000) + const ranges = rangesFromCalls(providerMock) + expect(ranges).toEqual([ + [16000, 19999], + [20000, 29999], + [30000, 35000], + ]) + }) + }) +}) diff --git a/packages/discovery/src/discovery/provider/DiscoveryProvider.ts b/packages/discovery/src/discovery/provider/DiscoveryProvider.ts index 35f7a161..a88dfb8c 100644 --- a/packages/discovery/src/discovery/provider/DiscoveryProvider.ts +++ b/packages/discovery/src/discovery/provider/DiscoveryProvider.ts @@ -6,6 +6,7 @@ import { EthereumAddress } from '../../utils/EthereumAddress' import { EtherscanLikeClient } from '../../utils/EtherscanLikeClient' import { Hash256 } from '../../utils/Hash256' import { UnixTime } from '../../utils/UnixTime' +import { DiscoveryLogger } from '../DiscoveryLogger' import { jsonToHumanReadableAbi } from './jsonToHumanReadableAbi' export interface ContractMetadata { @@ -29,6 +30,8 @@ export class DiscoveryProvider { constructor( private readonly provider: providers.Provider, private readonly etherscanLikeClient: EtherscanLikeClient, + private readonly logger: DiscoveryLogger, + private readonly getLogsMaxRange?: number, ) {} async call( @@ -56,13 +59,55 @@ export class DiscoveryProvider { return Bytes.fromHex(result) } - async getLogs( + public async getLogs( address: EthereumAddress, topics: (string | string[])[], fromBlock: number, toBlock: number, ): Promise { - return this.provider.getLogs({ + if (fromBlock > toBlock) { + throw new Error( + `fromBlock (${fromBlock}) can't be bigger than toBlock (${toBlock})`, + ) + } + + if (this.getLogsMaxRange === undefined) { + return await this.getLogsBatch(address, topics, fromBlock, toBlock) + } + + // To support efficient caching, we divide the requested blocks range into + // sequential boundaries of `maxRange` size, e.g [0,10k-1], [10k, 20k-1], ... + // Otherwise ranges would depend on `fromBlock` and even small change to it + // would make the previous cache useless. + + // Let's start with the deployment block number if it's higher than fromBlock + const { blockNumber: deploymentBlockNumber } = + await this.getDeploymentInfo(address) + + const maxRange = this.getLogsMaxRange + const allLogs: providers.Log[][] = [] + + let start = Math.max(fromBlock, deploymentBlockNumber) + do { + const curBoundaryStart = Math.floor(start / maxRange) * maxRange + const curBoundaryEnd = curBoundaryStart + maxRange - 1 // getLogs 'to' is inclusive! + const end = Math.min(curBoundaryEnd, toBlock) + const logs = await this.getLogsBatch(address, topics, start, end) + allLogs.push(logs) + start = end + 1 + } while (start <= toBlock) + + return allLogs.flat() + } + + public async getLogsBatch( + address: EthereumAddress, + topics: (string | string[])[], + fromBlock: number, + toBlock: number, + ): Promise { + this.logger.logFetchingEvents(fromBlock, toBlock) + return await this.provider.getLogs({ address: address.toString(), fromBlock, toBlock, @@ -113,12 +158,18 @@ export class DiscoveryProvider { return this.etherscanLikeClient.getFirstTxTimestamp(address) } - async getDeploymentTimestamp(address: EthereumAddress): Promise { + async getDeploymentInfo(address: EthereumAddress): Promise<{ + blockNumber: number + timestamp: UnixTime + }> { const txHash = await this.getContractDeploymentTx(address) const tx = await this.provider.getTransaction(txHash.toString()) assert(tx.blockNumber, 'Transaction returned without a block number.') const block = await this.provider.getBlock(tx.blockNumber) - return new UnixTime(block.timestamp) + return { + blockNumber: tx.blockNumber, + timestamp: new UnixTime(block.timestamp), + } } async getBlockNumber(): Promise { diff --git a/packages/discovery/src/discovery/provider/ProviderWithCache.ts b/packages/discovery/src/discovery/provider/ProviderWithCache.ts index d7a287d4..4bca8712 100644 --- a/packages/discovery/src/discovery/provider/ProviderWithCache.ts +++ b/packages/discovery/src/discovery/provider/ProviderWithCache.ts @@ -1,3 +1,4 @@ +import { createHash } from 'crypto' import { providers } from 'ethers' import { Bytes } from '../../utils/Bytes' @@ -5,6 +6,7 @@ import { ChainId } from '../../utils/ChainId' import { EthereumAddress } from '../../utils/EthereumAddress' import { EtherscanLikeClient } from '../../utils/EtherscanLikeClient' import { Hash256 } from '../../utils/Hash256' +import { DiscoveryLogger } from '../DiscoveryLogger' import { isRevert } from '../utils/isRevert' import { ContractMetadata, DiscoveryProvider } from './DiscoveryProvider' import { ProviderCache } from './ProviderCache' @@ -17,9 +19,11 @@ export class ProviderWithCache extends DiscoveryProvider { constructor( provider: providers.Provider, etherscanClient: EtherscanLikeClient, + logger: DiscoveryLogger, chainId: ChainId, + getLogsMaxRange?: number, ) { - super(provider, etherscanClient) + super(provider, etherscanClient, logger, getLogsMaxRange) this.cache = new ProviderCache(chainId) } @@ -86,16 +90,20 @@ export class ProviderWithCache extends DiscoveryProvider { ) } - override async getLogs( + override async getLogsBatch( address: EthereumAddress, topics: string[][], fromBlock: number, - blockNumber: number, + toBlock: number, ): Promise { + const topicsHash: string = createHash('sha256') + .update(JSON.stringify(topics)) + .digest('hex') + return this.cacheOrFetch( - `blocks/${blockNumber}`, - `getLogs.${address.toString()}.${JSON.stringify(topics)}.${fromBlock}`, - () => super.getLogs(address, topics, fromBlock, blockNumber), + `logs/${address.toString()}`, + `getLogs.${fromBlock}.${toBlock}.${topicsHash}`, + () => super.getLogsBatch(address, topics, fromBlock, toBlock), identity, identity, ) diff --git a/packages/discovery/src/discovery/runDiscovery.ts b/packages/discovery/src/discovery/runDiscovery.ts index 8a8c6739..d32c6530 100644 --- a/packages/discovery/src/discovery/runDiscovery.ts +++ b/packages/discovery/src/discovery/runDiscovery.ts @@ -41,6 +41,7 @@ export async function runDiscovery( projectConfig, logger, blockNumber, + config.getLogsMaxRange, ) await saveDiscoveryResult( result, @@ -67,12 +68,19 @@ export async function dryRunDiscovery( ) const [discovered, discoveredYesterday] = await Promise.all([ - justDiscover(provider, etherscanClient, projectConfig, blockNumber), + justDiscover( + provider, + etherscanClient, + projectConfig, + blockNumber, + config.getLogsMaxRange, + ), justDiscover( provider, etherscanClient, projectConfig, blockNumberYesterday, + config.getLogsMaxRange, ), ]) @@ -94,6 +102,7 @@ export async function justDiscover( etherscanClient: EtherscanLikeClient, config: DiscoveryConfig, blockNumber: number, + getLogsMaxRange?: number, ): Promise { const result = await discover( provider, @@ -101,6 +110,7 @@ export async function justDiscover( config, DiscoveryLogger.CLI, blockNumber, + getLogsMaxRange, ) const { name } = config @@ -124,11 +134,14 @@ export async function discover( config: DiscoveryConfig, logger: DiscoveryLogger, blockNumber: number, + getLogsMaxRange?: number, ): Promise { const discoveryProvider = new ProviderWithCache( provider, etherscanClient, + logger, config.chainId, + getLogsMaxRange, ) const proxyDetector = new ProxyDetector(discoveryProvider, logger) const sourceCodeService = new SourceCodeService(discoveryProvider) diff --git a/packages/discovery/src/singleDiscovery.ts b/packages/discovery/src/singleDiscovery.ts index fa433cb6..f8a18232 100644 --- a/packages/discovery/src/singleDiscovery.ts +++ b/packages/discovery/src/singleDiscovery.ts @@ -48,6 +48,7 @@ export async function singleDiscovery( projectConfig, DiscoveryLogger.CLI, blockNumber, + chainConfig.rpcGetLogsMaxRange, ) const discoveryOutput = toDiscoveryOutput(