This repository has been archived by the owner on Apr 26, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Fetch logs in batches, with cache-friendly ranges (#43)
* Fetch logs in batches, with cache-friendly ranges * Configure GETLOGS_MAX_RANGE for each project in env * Simplify getLogs implementation * Rename variable * Fix test * A tiny refactor * Tiny fix (let->const) * Add comment * Add changeset file
- Loading branch information
Showing
11 changed files
with
301 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| --- | ||
| '@l2beat/discovery': minor | ||
| --- | ||
|
|
||
| Support fetching logs in batches |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
152 changes: 152 additions & 0 deletions
152
packages/discovery/src/discovery/provider/DiscoveryProvider.test.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,152 @@ | ||
| import { expect, mockFn, MockObject, mockObject } from 'earl' | ||
| import { providers } from 'ethers' | ||
|
|
||
| import { EthereumAddress } from '../../utils/EthereumAddress' | ||
| import { EtherscanLikeClient } from '../../utils/EtherscanLikeClient' | ||
| import { DiscoveryLogger } from '../DiscoveryLogger' | ||
| import { DiscoveryProvider } from './DiscoveryProvider' | ||
|
|
||
| const rangesFromCalls = (provider: MockObject<providers.Provider>) => | ||
| provider.getLogs.calls.map((call) => [ | ||
| call.args[0].fromBlock, | ||
| call.args[0].toBlock, | ||
| ]) | ||
|
|
||
| const GETLOGS_MAX_RANGE = 10000 | ||
|
|
||
| describe(DiscoveryProvider.name, () => { | ||
| describe(DiscoveryProvider.prototype.getLogs.name, () => { | ||
| const etherscanLikeClientMock = mockObject<EtherscanLikeClient>({}) | ||
| const address = EthereumAddress.random() | ||
| const topics = ['testTopic'] | ||
| let providerMock: MockObject<providers.Provider> | ||
| let discoveryProviderMock: DiscoveryProvider | ||
|
|
||
| beforeEach(() => { | ||
| providerMock = mockObject<providers.Provider>({ | ||
| getLogs: mockFn().resolvesTo([]), | ||
| }) | ||
| discoveryProviderMock = new DiscoveryProvider( | ||
| providerMock, | ||
| etherscanLikeClientMock, | ||
| DiscoveryLogger.SILENT, | ||
| GETLOGS_MAX_RANGE, | ||
| ) | ||
| discoveryProviderMock.getDeploymentInfo = mockFn().resolvesTo({ | ||
| blockNumber: 0, | ||
| timestamp: 0, | ||
| }) | ||
| }) | ||
|
|
||
| it('handles simple range', async () => { | ||
| await discoveryProviderMock.getLogs(address, topics, 5000, 35000) | ||
| const ranges = rangesFromCalls(providerMock) | ||
| expect(ranges).toEqual([ | ||
| [5000, 9999], | ||
| [10000, 19999], | ||
| [20000, 29999], | ||
| [30000, 35000], | ||
| ]) | ||
| }) | ||
|
|
||
| it('handles range at boundaries', async () => { | ||
| await discoveryProviderMock.getLogs(address, topics, 10000, 39999) | ||
| const ranges = rangesFromCalls(providerMock) | ||
| expect(ranges).toEqual([ | ||
| [10000, 19999], | ||
| [20000, 29999], | ||
| [30000, 39999], | ||
| ]) | ||
| }) | ||
|
|
||
| it('handles range at +/- 1 of boundaries', async () => { | ||
| await discoveryProviderMock.getLogs(address, topics, 9999, 30000) | ||
| const ranges = rangesFromCalls(providerMock) | ||
| expect(ranges).toEqual([ | ||
| [9999, 9999], | ||
| [10000, 19999], | ||
| [20000, 29999], | ||
| [30000, 30000], | ||
| ]) | ||
| }) | ||
|
|
||
| it('handles range where from and to are equal and multiply or range', async () => { | ||
| await discoveryProviderMock.getLogs(address, topics, 10000, 10000) | ||
| const ranges = rangesFromCalls(providerMock) | ||
| expect(ranges).toEqual([[10000, 10000]]) | ||
| }) | ||
|
|
||
| it('handles range where from and to are -1 of multiply or range', async () => { | ||
| await discoveryProviderMock.getLogs(address, topics, 9999, 9999) | ||
| const ranges = rangesFromCalls(providerMock) | ||
| expect(ranges).toEqual([[9999, 9999]]) | ||
| }) | ||
|
|
||
| it('handles range where from and to are +1 of multiply or range', async () => { | ||
| await discoveryProviderMock.getLogs(address, topics, 10001, 10001) | ||
| const ranges = rangesFromCalls(providerMock) | ||
| expect(ranges).toEqual([[10001, 10001]]) | ||
| }) | ||
|
|
||
| it('handles range [0,0]', async () => { | ||
| await discoveryProviderMock.getLogs(address, topics, 0, 0) | ||
| const ranges = rangesFromCalls(providerMock) | ||
| expect(ranges).toEqual([[0, 0]]) | ||
| }) | ||
|
|
||
| it('handles range [1,1]', async () => { | ||
| await discoveryProviderMock.getLogs(address, topics, 1, 1) | ||
| const ranges = rangesFromCalls(providerMock) | ||
| expect(ranges).toEqual([[1, 1]]) | ||
| }) | ||
|
|
||
| it('handles range inside boundaries', async () => { | ||
| await discoveryProviderMock.getLogs(address, topics, 1400, 1600) | ||
| const ranges = rangesFromCalls(providerMock) | ||
| expect(ranges).toEqual([[1400, 1600]]) | ||
| }) | ||
|
|
||
| it('handles getLogsMaxRange undefined (no range)', async () => { | ||
| providerMock = mockObject<providers.Provider>({ | ||
| getLogs: mockFn().resolvesTo([]), | ||
| }) | ||
| discoveryProviderMock = new DiscoveryProvider( | ||
| providerMock, | ||
| etherscanLikeClientMock, | ||
| DiscoveryLogger.SILENT, | ||
| undefined, // PROVIDING UNDEFINED for getLogsMaxRange, so no batching | ||
| ) | ||
| discoveryProviderMock.getDeploymentInfo = mockFn().resolvesTo({ | ||
| blockNumber: 0, | ||
| timestamp: 0, | ||
| }) | ||
| await discoveryProviderMock.getLogs(address, topics, 5000, 35000) | ||
| const ranges = rangesFromCalls(providerMock) | ||
| expect(ranges).toEqual([[5000, 35000]]) | ||
| }) | ||
|
|
||
| it('starts with deployment block if bigger then fromBlock', async () => { | ||
| providerMock = mockObject<providers.Provider>({ | ||
| getLogs: mockFn().resolvesTo([]), | ||
| }) | ||
| discoveryProviderMock = new DiscoveryProvider( | ||
| providerMock, | ||
| etherscanLikeClientMock, | ||
| DiscoveryLogger.SILENT, | ||
| GETLOGS_MAX_RANGE, | ||
| ) | ||
| discoveryProviderMock.getDeploymentInfo = mockFn().resolvesTo({ | ||
| blockNumber: 16000, | ||
| timestamp: 0, | ||
| }) | ||
|
|
||
| await discoveryProviderMock.getLogs(address, topics, 5000, 35000) | ||
| const ranges = rangesFromCalls(providerMock) | ||
| expect(ranges).toEqual([ | ||
| [16000, 19999], | ||
| [20000, 29999], | ||
| [30000, 35000], | ||
| ]) | ||
| }) | ||
| }) | ||
| }) |
Oops, something went wrong.