diff --git a/packages/example/.mocharc.json b/packages/example/.mocharc.json deleted file mode 100644 index 862b85f9..00000000 --- a/packages/example/.mocharc.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "spec": "src/**/*.test.ts", - "require": ["esbuild-register"], - "watchExtensions": "ts", - "extension": "ts", - "reporterOption": "maxDiffSize=0" -} diff --git a/packages/example/src/Application.ts b/packages/example/src/Application.ts deleted file mode 100644 index f238e7ff..00000000 --- a/packages/example/src/Application.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { Logger } from '@l2beat/backend-tools' -import { BaseIndexer, Retries } from '@l2beat/uif' - -import { Config } from './Config' -import { AB_BC_Indexer } from './indexers/AB_BC_Indexer' -import { ABC_Indexer } from './indexers/ABC_Indexer' -import { BalanceIndexer } from './indexers/BalanceIndexer' -import { BlockNumberIndexer } from './indexers/BlockNumberIndexer' -import { FakeClockIndexer } from './indexers/FakeClockIndexer' -import { TvlIndexer } from './indexers/TvlIndexer' -import { AB_BC_Repository } from './repositories/AB_BC_Repository' -import { ABC_Repository } from './repositories/ABC_Repository' -import { BalanceRepository } from './repositories/BalanceRepository' -import { BlockNumberRepository } from './repositories/BlockNumberRepository' -import { TvlRepository } from './repositories/TvlRepository' - -interface Module { - start: () => Promise -} - -export class Application { - start: () => Promise - - constructor(config: Config) { - const logger = new Logger({ - logLevel: 'DEBUG', - format: 'pretty', - colors: true, - utc: true, - }) - - const modules = [ - createMainModule(config, logger), - createABCModule(config, logger), - ] - - this.start = async (): Promise => { - console.log(`Application started: ${config.name}`) - await Promise.all(modules.map((module) => module?.start())) - } - } -} - -function createABCModule(config: Config, logger: Logger): Module | undefined { - if (!config.modules.abc) { - return undefined - } - - const blockNumberRepository = new BlockNumberRepository() - const abc_repository = new ABC_Repository() - const ab_bc_repository = new AB_BC_Repository() - - const fakeClockIndexer = new FakeClockIndexer( - logger.configure({ logLevel: 'NONE' }), - ) - const blockNumberIndexer = new BlockNumberIndexer( - logger.configure({ logLevel: 'NONE' }), - fakeClockIndexer, - blockNumberRepository, - ) - const abc_Indexer = new ABC_Indexer( - logger, - blockNumberIndexer, - abc_repository, - ) - const ab_bc_indexer = new AB_BC_Indexer( - logger, - abc_Indexer, - ab_bc_repository, - abc_repository, - ) - - return { - start: async (): Promise => { - console.log(`Module started: ABCModule`) - - await fakeClockIndexer.start() - await blockNumberIndexer.start() - await abc_Indexer.start() - await ab_bc_indexer.start() - }, - } -} - -function createMainModule(config: Config, logger: Logger): Module | undefined { - if (!config.modules.main) { - return undefined - } - - const blockNumberRepository = new BlockNumberRepository() - const balanceRepository = new BalanceRepository() - const tvlRepository = new TvlRepository() - - BaseIndexer.GET_DEFAULT_RETRY_STRATEGY = () => - Retries.exponentialBackOff({ - initialTimeoutMs: 100, - maxAttempts: 10, - maxTimeoutMs: 60 * 1000, - }) - - const fakeClockIndexer = new FakeClockIndexer(logger) - const blockNumberIndexer = new BlockNumberIndexer( - logger, - fakeClockIndexer, - blockNumberRepository, - ) - const balanceIndexer = new BalanceIndexer( - logger, - blockNumberIndexer, - balanceRepository, - ) - const tvlIndexer = new TvlIndexer(logger, balanceIndexer, tvlRepository) - - return { - start: async (): Promise => { - await Promise.resolve() - console.log(`Application started: MainModule`) - - await fakeClockIndexer.start() - await blockNumberIndexer.start() - await balanceIndexer.start() - await tvlIndexer.start() - }, - } -} diff --git a/packages/example/src/Config.ts b/packages/example/src/Config.ts deleted file mode 100644 index cfbbd76e..00000000 --- a/packages/example/src/Config.ts +++ /dev/null @@ -1,17 +0,0 @@ -export interface Config { - name: string - modules: { - main: boolean - abc: boolean - } -} - -export function getConfig(): Config { - return { - name: 'uif-example', - modules: { - main: false, - abc: true, - }, - } -} diff --git a/packages/example/src/index.test.ts b/packages/example/src/index.test.ts deleted file mode 100644 index 7736fd3e..00000000 --- a/packages/example/src/index.test.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { expect } from 'earl' - -describe('plus operator', () => { - it('adds two numbers', () => { - expect(1 + 2).toEqual(3) - }) -}) diff --git a/packages/example/src/indexers/ABC_Indexer.ts b/packages/example/src/indexers/ABC_Indexer.ts deleted file mode 100644 index 3c5b1bae..00000000 --- a/packages/example/src/indexers/ABC_Indexer.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { Logger } from '@l2beat/backend-tools' -import { SliceIndexer, SliceState, SliceUpdate } from '@l2beat/uif' - -import { ABC_Repository } from '../repositories/ABC_Repository' -import { BlockNumberIndexer } from './BlockNumberIndexer' - -export class ABC_Indexer extends SliceIndexer { - constructor( - logger: Logger, - blockNumberIndexer: BlockNumberIndexer, - private readonly abc_repository: ABC_Repository, - private readonly tokens = ['A', 'B', 'C'], - ) { - super(logger, [blockNumberIndexer]) - } - - override getExpectedSlices(): string[] { - return this.tokens - } - - override async getSliceStates(): Promise { - const tokenHeights = await this.abc_repository.getSliceHeights() - const states = [...tokenHeights.entries()].map( - ([token, height]): SliceState => ({ - sliceHash: token, - height, - }), - ) - return states - } - - override async removeSlices(hashes: string[]): Promise { - return await this.abc_repository.removeSlices(hashes) - } - - override async updateSlices(updates: SliceUpdate[]): Promise { - let minHeight = Infinity - for (const update of updates) { - const balances = await this.abc_repository.getSliceData(update.sliceHash) - - for (let i = update.from; i <= update.to; i++) { - balances.set(i, i * 2) - } - if (update.to < minHeight) { - minHeight = update.to - } - - await this.abc_repository.setSliceData(update.sliceHash, balances) - await this.abc_repository.setSliceHeight(update.sliceHash, update.to) - } - return minHeight - } - - override async getMainSafeHeight(): Promise { - return await this.abc_repository.getSafeHeight() - } - - override async setMainSafeHeight(height: number): Promise { - await this.abc_repository.setSafeHeight(height) - } - - override invalidate(targetHeight: number): Promise { - return Promise.resolve(targetHeight) - } -} diff --git a/packages/example/src/indexers/AB_BC_Indexer.ts b/packages/example/src/indexers/AB_BC_Indexer.ts deleted file mode 100644 index 3aa3c997..00000000 --- a/packages/example/src/indexers/AB_BC_Indexer.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { Logger } from '@l2beat/backend-tools' -import { SliceIndexer, SliceState, SliceUpdate } from '@l2beat/uif' - -import { AB_BC_Repository } from '../repositories/AB_BC_Repository' -import { ABC_Repository } from '../repositories/ABC_Repository' -import { ABC_Indexer } from './ABC_Indexer' - -export class AB_BC_Indexer extends SliceIndexer { - private readonly slices = new Map() - - constructor( - logger: Logger, - abc_indexer: ABC_Indexer, - private readonly ab_bc_repository: AB_BC_Repository, - private readonly abc_repository: ABC_Repository, - private readonly sums = [ - ['A', 'B'], - ['B', 'C'], - ], - ) { - super(logger, [abc_indexer]) - this.slices = new Map( - this.sums.map((sum) => [sum.join('+'), sum]), - ) - } - - override getExpectedSlices(): string[] { - return [...this.slices.keys()] - } - - override async getSliceStates(): Promise { - const sliceHeights = await this.ab_bc_repository.getSliceHeights() - const states = [...sliceHeights.entries()].map( - ([sliceHash, height]): SliceState => ({ sliceHash, height }), - ) - return Promise.resolve(states) - } - - override async removeSlices(hashes: string[]): Promise { - await this.ab_bc_repository.removeSlices(hashes) - } - - override async updateSlices(updates: SliceUpdate[]): Promise { - let minHeight = Infinity - for (const update of updates) { - const sumMap = await this.ab_bc_repository.getSliceData(update.sliceHash) - - for (let i = update.from; i <= update.to; i++) { - const tokens = this.slices.get(update.sliceHash) ?? [] - const values = await Promise.all( - tokens.map((token) => this.abc_repository.getTokenBalance(token, i)), - ) - sumMap.set( - i, - values.reduce((a, b) => a + b, 0), - ) - } - if (update.to < minHeight) { - minHeight = update.to - } - - await this.ab_bc_repository.setSliceData(update.sliceHash, sumMap) - await this.ab_bc_repository.setSliceHeight(update.sliceHash, update.to) - } - return Promise.resolve(minHeight) - } - - override async getMainSafeHeight(): Promise { - return await this.ab_bc_repository.getSafeHeight() - } - - override async setMainSafeHeight(height: number): Promise { - await this.ab_bc_repository.setSafeHeight(height) - } - - override async invalidate(targetHeight: number): Promise { - return Promise.resolve(targetHeight) - } -} diff --git a/packages/example/src/indexers/BalanceIndexer.ts b/packages/example/src/indexers/BalanceIndexer.ts deleted file mode 100644 index c919b237..00000000 --- a/packages/example/src/indexers/BalanceIndexer.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { Logger } from '@l2beat/backend-tools' -import { ChildIndexer } from '@l2beat/uif' -import { setTimeout } from 'timers/promises' - -import { BalanceRepository } from '../repositories/BalanceRepository' -import { BlockNumberIndexer } from './BlockNumberIndexer' - -export class BalanceIndexer extends ChildIndexer { - constructor( - logger: Logger, - blockNumberIndexer: BlockNumberIndexer, - private readonly balanceRepository: BalanceRepository, - ) { - super(logger, [blockNumberIndexer]) - } - - override async update(from: number, to: number): Promise { - await setTimeout(1_000) - if (Math.random() < 0.5) { - this.logger.info('BalanceIndexer: height decreased') - return Math.max(from - 500, 0) - } - to = Math.min(from + 100, to) - return to - } - - override async invalidate(targetHeight: number): Promise { - return Promise.resolve(targetHeight) - } - - override async getSafeHeight(): Promise { - const height = await this.balanceRepository.getLastSynced() - return height ?? 0 - } - - override async setSafeHeight(height: number): Promise { - return this.balanceRepository.setLastSynced(height) - } -} diff --git a/packages/example/src/indexers/BlockNumberIndexer.ts b/packages/example/src/indexers/BlockNumberIndexer.ts deleted file mode 100644 index a4c239f6..00000000 --- a/packages/example/src/indexers/BlockNumberIndexer.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { Logger } from '@l2beat/backend-tools' -import { ChildIndexer } from '@l2beat/uif' -import { setTimeout } from 'timers/promises' - -import { BlockNumberRepository } from '../repositories/BlockNumberRepository' -import { FakeClockIndexer } from './FakeClockIndexer' - -export class BlockNumberIndexer extends ChildIndexer { - constructor( - logger: Logger, - fakeClockIndexer: FakeClockIndexer, - private readonly blockNumberRepository: BlockNumberRepository, - ) { - super(logger, [fakeClockIndexer]) - } - - override async update(from: number, targetHeight: number): Promise { - await setTimeout(2_000) - if (Math.random() < 0.5) { - throw new Error('Random error while updating') - } - return targetHeight - } - - override async invalidate(targetHeight: number): Promise { - if (Math.random() < 0.5) { - throw new Error('Random error while invalidating') - } - return Promise.resolve(targetHeight) - } - - override async getSafeHeight(): Promise { - const height = await this.blockNumberRepository.getLastSynced() - return height ?? 0 - } - - override async setSafeHeight(height: number): Promise { - return this.blockNumberRepository.setLastSynced(height) - } -} diff --git a/packages/example/src/indexers/FakeClockIndexer.ts b/packages/example/src/indexers/FakeClockIndexer.ts deleted file mode 100644 index c6090b66..00000000 --- a/packages/example/src/indexers/FakeClockIndexer.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { RootIndexer } from '@l2beat/uif' - -export class FakeClockIndexer extends RootIndexer { - private height = 100 - - override async start(): Promise { - await super.start() - setInterval(() => this.requestTick(), 1_000) - } - - override async tick(): Promise { - if (Math.random() < 0.05) { - this.height = Math.max(this.height - 50, 0) - this.logger.info('FakeClockIndexer: height decreased') - } else { - this.height += 10 - } - return Promise.resolve(this.height) - } -} diff --git a/packages/example/src/indexers/HourlyIndexer.ts b/packages/example/src/indexers/HourlyIndexer.ts deleted file mode 100644 index 2fa9bef2..00000000 --- a/packages/example/src/indexers/HourlyIndexer.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { RootIndexer } from '@l2beat/uif' - -export class HourlyIndexer extends RootIndexer { - override async start(): Promise { - await super.start() - setInterval(() => this.requestTick(), 60 * 1000) - } - - async tick(): Promise { - const hourInMs = 60 * 60 * 1000 - const time = (new Date().getTime() % hourInMs) * hourInMs - return Promise.resolve(time) - } -} diff --git a/packages/example/src/indexers/TvlIndexer.ts b/packages/example/src/indexers/TvlIndexer.ts deleted file mode 100644 index 9406f392..00000000 --- a/packages/example/src/indexers/TvlIndexer.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { Logger } from '@l2beat/backend-tools' -import { ChildIndexer } from '@l2beat/uif' -import { setTimeout } from 'timers/promises' - -import { TvlRepository } from '../repositories/TvlRepository' -import { BalanceIndexer } from './BalanceIndexer' - -export class TvlIndexer extends ChildIndexer { - height = 0 - - constructor( - logger: Logger, - balanceIndexer: BalanceIndexer, - private readonly tvlRepository: TvlRepository, - ) { - super(logger, [balanceIndexer]) - } - - override async update(from: number, to: number): Promise { - await setTimeout(500) - to = Math.min(from + 10, to) - this.height = to - return to - } - - override async invalidate(targetHeight: number): Promise { - const newHeight = Math.max(this.height - 5, targetHeight) - this.height = newHeight - return await Promise.resolve(newHeight) - } - - override async getSafeHeight(): Promise { - const height = await this.tvlRepository.getLastSynced() - return height ?? 0 - } - - override async setSafeHeight(height: number): Promise { - await this.tvlRepository.setLastSynced(height) - } -} diff --git a/packages/example/src/repositories/ABC_Repository.ts b/packages/example/src/repositories/ABC_Repository.ts deleted file mode 100644 index e3c22cd2..00000000 --- a/packages/example/src/repositories/ABC_Repository.ts +++ /dev/null @@ -1,46 +0,0 @@ -type SliceHeights = Map -type SliceData = Map - -export class ABC_Repository { - private readonly sliceHeights = new Map() - private readonly slicesData = new Map>() - private safeHeight = 0 - - getSliceHeights(): Promise { - return Promise.resolve(this.sliceHeights) - } - getSliceData(hash: string): Promise { - const sliceData = this.slicesData.get(hash) ?? new Map() - return Promise.resolve(sliceData) - } - async removeSlices(hashes: string[]): Promise { - await Promise.resolve() - for (const hash of hashes) { - this.sliceHeights.delete(hash) - this.slicesData.delete(hash) - } - } - async setSliceHeight(hash: string, height: number): Promise { - await Promise.resolve() - this.sliceHeights.set(hash, height) - } - async setSliceData(hash: string, data: Map): Promise { - await Promise.resolve() - this.slicesData.set(hash, data) - } - getSafeHeight(): Promise { - return Promise.resolve(this.safeHeight) - } - async setSafeHeight(height: number): Promise { - await Promise.resolve() - this.safeHeight = height - } - - getTokenBalance(token: string, blockNumber: number): Promise { - const balance = this.slicesData.get(token)?.get(blockNumber) - if (balance === undefined) { - throw new Error(`No balance of ${token} for ${blockNumber}`) - } - return Promise.resolve(balance) - } -} diff --git a/packages/example/src/repositories/AB_BC_Repository.ts b/packages/example/src/repositories/AB_BC_Repository.ts deleted file mode 100644 index dd44d217..00000000 --- a/packages/example/src/repositories/AB_BC_Repository.ts +++ /dev/null @@ -1,38 +0,0 @@ -type SliceHeights = Map -type SliceData = Map - -export class AB_BC_Repository { - private readonly sliceHeights = new Map() - private readonly slicesData = new Map>() - private safeHeight = 0 - - getSliceHeights(): Promise { - return Promise.resolve(this.sliceHeights) - } - getSliceData(hash: string): Promise { - const sliceData = this.slicesData.get(hash) ?? new Map() - return Promise.resolve(sliceData) - } - async removeSlices(hashes: string[]): Promise { - await Promise.resolve() - for (const hash of hashes) { - this.sliceHeights.delete(hash) - this.slicesData.delete(hash) - } - } - async setSliceHeight(hash: string, height: number): Promise { - await Promise.resolve() - this.sliceHeights.set(hash, height) - } - async setSliceData(hash: string, data: Map): Promise { - await Promise.resolve() - this.slicesData.set(hash, data) - } - getSafeHeight(): Promise { - return Promise.resolve(this.safeHeight) - } - async setSafeHeight(height: number): Promise { - await Promise.resolve() - this.safeHeight = height - } -} diff --git a/packages/example/src/repositories/BalanceRepository.ts b/packages/example/src/repositories/BalanceRepository.ts deleted file mode 100644 index 0e15caee..00000000 --- a/packages/example/src/repositories/BalanceRepository.ts +++ /dev/null @@ -1,12 +0,0 @@ -export class BalanceRepository { - private lastSynced: number | undefined = undefined - - async getLastSynced(): Promise { - return Promise.resolve(this.lastSynced) - } - - async setLastSynced(lastSynced: number): Promise { - this.lastSynced = lastSynced - return Promise.resolve() - } -} diff --git a/packages/example/src/repositories/BlockNumberRepository.ts b/packages/example/src/repositories/BlockNumberRepository.ts deleted file mode 100644 index c1a20bb0..00000000 --- a/packages/example/src/repositories/BlockNumberRepository.ts +++ /dev/null @@ -1,12 +0,0 @@ -export class BlockNumberRepository { - private lastSynced: number | undefined = undefined - - async getLastSynced(): Promise { - return Promise.resolve(this.lastSynced) - } - - async setLastSynced(lastSynced: number): Promise { - this.lastSynced = lastSynced - return Promise.resolve() - } -} diff --git a/packages/example/src/repositories/TvlRepository.ts b/packages/example/src/repositories/TvlRepository.ts deleted file mode 100644 index 4d85b6b6..00000000 --- a/packages/example/src/repositories/TvlRepository.ts +++ /dev/null @@ -1,12 +0,0 @@ -export class TvlRepository { - private lastSynced: number | undefined = undefined - - async getLastSynced(): Promise { - return Promise.resolve(this.lastSynced) - } - - async setLastSynced(lastSynced: number): Promise { - this.lastSynced = lastSynced - return Promise.resolve() - } -} diff --git a/packages/example/.eslintrc b/packages/uif-example/.eslintrc similarity index 100% rename from packages/example/.eslintrc rename to packages/uif-example/.eslintrc diff --git a/packages/example/.prettierignore b/packages/uif-example/.prettierignore similarity index 100% rename from packages/example/.prettierignore rename to packages/uif-example/.prettierignore diff --git a/packages/example/.prettierrc b/packages/uif-example/.prettierrc similarity index 100% rename from packages/example/.prettierrc rename to packages/uif-example/.prettierrc diff --git a/packages/example/package.json b/packages/uif-example/package.json similarity index 94% rename from packages/example/package.json rename to packages/uif-example/package.json index 6ecd57d0..4bd9399c 100644 --- a/packages/example/package.json +++ b/packages/uif-example/package.json @@ -1,5 +1,5 @@ { - "name": "example", + "name": "uif-example", "license": "MIT", "version": "0.1.0", "private": true, diff --git a/packages/uif-example/src/Application.ts b/packages/uif-example/src/Application.ts new file mode 100644 index 00000000..3088053d --- /dev/null +++ b/packages/uif-example/src/Application.ts @@ -0,0 +1,89 @@ +import { Logger } from '@l2beat/backend-tools' + +import { BlockIndexer } from './blocks/BlockIndexer' +import { BlockIndexerRepository } from './blocks/BlockIndexerRepository' +import { BlockRepository } from './blocks/BlockRepository' +import { BlockService } from './blocks/BlockService' +import { HourlyIndexer } from './HourlyIndexer' +import { PriceIndexer } from './prices/PriceIndexer' +import { PriceIndexerRepository } from './prices/PriceIndexerRepository' +import { PriceRepository } from './prices/PriceRepository' +import { PriceService } from './prices/PriceService' +import { msToHours, ONE_HOUR_MS } from './utils' + +export class Application { + start: () => Promise + + constructor() { + const logger = Logger.DEBUG + + const hourlyIndexer = new HourlyIndexer(logger) + + const priceService = new PriceService(logger) + const priceRepository = new PriceRepository() + const priceIndexerRepository = new PriceIndexerRepository() + + const ethereumPriceIndexer = new PriceIndexer( + 'price-ethereum', + priceService, + priceRepository, + priceIndexerRepository, + hourlyIndexer, + logger, + [ + { + // could be a hash of properties & minHeight instead + id: 'eth-ethereum', + properties: { tokenSymbol: 'ETH', apiId: 'ethereum' }, + minHeight: msToHours(Date.now() - 48 * ONE_HOUR_MS), + maxHeight: null, + }, + { + id: 'weth-ethereum', + properties: { tokenSymbol: 'WETH', apiId: 'ethereum' }, + minHeight: msToHours(Date.now() - 32 * ONE_HOUR_MS), + maxHeight: null, + }, + ], + ) + const bitcoinPriceIndexer = new PriceIndexer( + 'price-bitcoin', + priceService, + priceRepository, + priceIndexerRepository, + hourlyIndexer, + logger, + [ + { + id: 'btc-bitcoin', + properties: { tokenSymbol: 'BTC', apiId: 'bitcoin' }, + minHeight: msToHours(Date.now() - 72 * ONE_HOUR_MS), + maxHeight: null, + }, + ], + ) + + const blockService = new BlockService() + const blockRepository = new BlockRepository() + const blockIndexerRepository = new BlockIndexerRepository() + const blockIndexer = new BlockIndexer( + blockService, + blockRepository, + blockIndexerRepository, + msToHours(Date.now() - 72 * ONE_HOUR_MS), + hourlyIndexer, + logger, + ) + + this.start = async (): Promise => { + logger.for('Application').info('Starting') + + await hourlyIndexer.start() + await ethereumPriceIndexer.start() + await bitcoinPriceIndexer.start() + await blockIndexer.start() + + logger.for('Application').info('Started') + } + } +} diff --git a/packages/uif-example/src/HourlyIndexer.ts b/packages/uif-example/src/HourlyIndexer.ts new file mode 100644 index 00000000..6093b8cf --- /dev/null +++ b/packages/uif-example/src/HourlyIndexer.ts @@ -0,0 +1,16 @@ +import { RootIndexer } from '@l2beat/uif' + +import { ONE_HOUR_MS } from './utils' + +export class HourlyIndexer extends RootIndexer { + async initialize(): Promise { + setInterval(() => this.requestTick(), 10 * 1000) + return this.tick() + } + + async tick(): Promise { + const now = new Date().getTime() + const hours = Math.floor(now / ONE_HOUR_MS) + return Promise.resolve(hours) + } +} diff --git a/packages/uif-example/src/blocks/BlockIndexer.ts b/packages/uif-example/src/blocks/BlockIndexer.ts new file mode 100644 index 00000000..d0641cf0 --- /dev/null +++ b/packages/uif-example/src/blocks/BlockIndexer.ts @@ -0,0 +1,45 @@ +import { Logger } from '@l2beat/backend-tools' +import { ChildIndexer, IndexerOptions } from '@l2beat/uif' + +import { HourlyIndexer } from '../HourlyIndexer' +import { ONE_HOUR_MS } from '../utils' +import { BlockIndexerRepository } from './BlockIndexerRepository' +import { BlockRepository } from './BlockRepository' +import { BlockService } from './BlockService' + +export class BlockIndexer extends ChildIndexer { + constructor( + private readonly blockService: BlockService, + private readonly blockRepository: BlockRepository, + private readonly blockIndexerRepository: BlockIndexerRepository, + private readonly minHeight: number, + hourlyIndexer: HourlyIndexer, + logger: Logger, + options?: IndexerOptions, + ) { + super(logger, [hourlyIndexer], options) + } + + override async initialize(): Promise { + const height = await this.blockIndexerRepository.loadHeight() + return height ?? this.minHeight - 1 + } + + override async setSafeHeight(height: number): Promise { + await this.blockIndexerRepository.saveHeight(height) + } + + override async update(from: number): Promise { + const timestamp = from * ONE_HOUR_MS + + const block = await this.blockService.getBlockNumberBefore(timestamp) + await this.blockRepository.save({ number: block, timestamp }) + + return from + } + + override async invalidate(targetHeight: number): Promise { + // We don't need to delete any data + return Promise.resolve(targetHeight) + } +} diff --git a/packages/uif-example/src/blocks/BlockIndexerRepository.ts b/packages/uif-example/src/blocks/BlockIndexerRepository.ts new file mode 100644 index 00000000..de63934f --- /dev/null +++ b/packages/uif-example/src/blocks/BlockIndexerRepository.ts @@ -0,0 +1,12 @@ +export class BlockIndexerRepository { + private height: number | undefined + + async loadHeight(): Promise { + return Promise.resolve(this.height) + } + + async saveHeight(height: number): Promise { + this.height = height + return Promise.resolve() + } +} diff --git a/packages/uif-example/src/blocks/BlockRepository.ts b/packages/uif-example/src/blocks/BlockRepository.ts new file mode 100644 index 00000000..f046278c --- /dev/null +++ b/packages/uif-example/src/blocks/BlockRepository.ts @@ -0,0 +1,8 @@ +export class BlockRepository { + private readonly blocks = new Map() + + async save(block: { number: number; timestamp: number }): Promise { + this.blocks.set(block.timestamp, block.number) + return Promise.resolve() + } +} diff --git a/packages/uif-example/src/blocks/BlockService.ts b/packages/uif-example/src/blocks/BlockService.ts new file mode 100644 index 00000000..d9825c72 --- /dev/null +++ b/packages/uif-example/src/blocks/BlockService.ts @@ -0,0 +1,8 @@ +import { setTimeout } from 'timers/promises' + +export class BlockService { + async getBlockNumberBefore(timestamp: number): Promise { + await setTimeout(200) + return Math.floor(timestamp / 123456) + } +} diff --git a/packages/example/src/index.ts b/packages/uif-example/src/index.ts similarity index 61% rename from packages/example/src/index.ts rename to packages/uif-example/src/index.ts index 5e062fb9..26f75c5f 100644 --- a/packages/example/src/index.ts +++ b/packages/uif-example/src/index.ts @@ -1,5 +1,4 @@ import { Application } from './Application' -import { getConfig } from './Config' main().catch((e) => { console.error(e) @@ -7,7 +6,6 @@ main().catch((e) => { }) async function main(): Promise { - const config = getConfig() - const application = new Application(config) + const application = new Application() await application.start() } diff --git a/packages/uif-example/src/prices/PriceConfig.ts b/packages/uif-example/src/prices/PriceConfig.ts new file mode 100644 index 00000000..1dbab632 --- /dev/null +++ b/packages/uif-example/src/prices/PriceConfig.ts @@ -0,0 +1,4 @@ +export interface PriceConfig { + tokenSymbol: string + apiId: string +} diff --git a/packages/uif-example/src/prices/PriceIndexer.ts b/packages/uif-example/src/prices/PriceIndexer.ts new file mode 100644 index 00000000..5abc032c --- /dev/null +++ b/packages/uif-example/src/prices/PriceIndexer.ts @@ -0,0 +1,97 @@ +import { Logger } from '@l2beat/backend-tools' +import { + Configuration, + IndexerOptions, + MultiIndexer, + RemovalConfiguration, + SavedConfiguration, + UpdateConfiguration, +} from '@l2beat/uif' + +import { HourlyIndexer } from '../HourlyIndexer' +import { ONE_HOUR_MS } from '../utils' +import { PriceConfig } from './PriceConfig' +import { PriceIndexerRepository } from './PriceIndexerRepository' +import { PriceRepository } from './PriceRepository' +import { PriceService } from './PriceService' + +export class PriceIndexer extends MultiIndexer { + private readonly apiId: string + + constructor( + private readonly indexerId: string, + private readonly priceService: PriceService, + private readonly priceRepository: PriceRepository, + private readonly priceIndexerRepository: PriceIndexerRepository, + hourlyIndexer: HourlyIndexer, + logger: Logger, + configurations: Configuration[], + options?: IndexerOptions, + ) { + super(logger, [hourlyIndexer], configurations, options) + this.apiId = getCommonApiId(configurations) + } + + override async multiInitialize(): Promise[]> { + return this.priceIndexerRepository.load(this.indexerId) + } + + override async multiUpdate( + from: number, + to: number, + configurations: UpdateConfiguration[], + ): Promise { + // we only query 24 hours at a time + const adjustedTo = Math.min(to, from + 23) + + const prices = await this.priceService.getHourlyPrices( + this.apiId, + from * ONE_HOUR_MS, + adjustedTo * ONE_HOUR_MS, + ) + + const dataToSave = configurations + // TODO: don't update currentHeight for configs that have data + // TODO: test data downloaded to middle of the range + .filter((c) => !c.hasData) + .flatMap((configuration) => { + return prices.map(({ timestamp, price }) => ({ + tokenSymbol: configuration.properties.tokenSymbol, + timestamp, + price, + })) + }) + await this.priceRepository.save(dataToSave) + + return adjustedTo + } + + override async removeData( + configurations: RemovalConfiguration[], + ): Promise { + for (const c of configurations) { + await this.priceRepository.deletePrices( + c.properties.tokenSymbol, + c.from, + c.to, + ) + } + } + + override async saveConfigurations( + configurations: SavedConfiguration[], + ): Promise { + return this.priceIndexerRepository.save(this.indexerId, configurations) + } +} + +function getCommonApiId(configurations: Configuration[]): string { + const apiId = configurations[0]?.properties.apiId + if (!apiId) { + throw new Error('At least one configuration is required') + } + if (configurations.some((c) => c.properties.apiId !== apiId)) { + throw new Error('All configurations must have the same apiId') + } + return apiId +} diff --git a/packages/uif-example/src/prices/PriceIndexerRepository.ts b/packages/uif-example/src/prices/PriceIndexerRepository.ts new file mode 100644 index 00000000..6fcac7fd --- /dev/null +++ b/packages/uif-example/src/prices/PriceIndexerRepository.ts @@ -0,0 +1,19 @@ +import { SavedConfiguration } from '@l2beat/uif' + +import { PriceConfig } from './PriceConfig' + +export class PriceIndexerRepository { + private data: Record[]> = {} + + async save( + indexerId: string, + configurations: SavedConfiguration[], + ): Promise { + this.data[indexerId] = configurations + return Promise.resolve() + } + + async load(indexerId: string): Promise[]> { + return Promise.resolve(this.data[indexerId] ?? []) + } +} diff --git a/packages/uif-example/src/prices/PriceRepository.ts b/packages/uif-example/src/prices/PriceRepository.ts new file mode 100644 index 00000000..2690a32c --- /dev/null +++ b/packages/uif-example/src/prices/PriceRepository.ts @@ -0,0 +1,38 @@ +export class PriceRepository { + data = new Map>() + + async save( + prices: { tokenSymbol: string; timestamp: number; price: number }[], + ): Promise { + for (const { tokenSymbol, timestamp, price } of prices) { + const tokenPrices = + this.data.get(tokenSymbol) ?? new Map() + this.data.set(tokenSymbol, tokenPrices) + + tokenPrices.set(timestamp, price) + } + return Promise.resolve() + } + + async deletePrices( + tokenSymbol: string, + fromTimestampInclusive: number, + toTimestampInclusive: number, + ): Promise { + const tokenPrices = this.data.get(tokenSymbol) + if (!tokenPrices) { + return + } + + for (const [timestamp] of tokenPrices) { + if ( + timestamp >= fromTimestampInclusive && + timestamp <= toTimestampInclusive + ) { + tokenPrices.delete(timestamp) + } + } + + return Promise.resolve() + } +} diff --git a/packages/uif-example/src/prices/PriceService.ts b/packages/uif-example/src/prices/PriceService.ts new file mode 100644 index 00000000..0026a094 --- /dev/null +++ b/packages/uif-example/src/prices/PriceService.ts @@ -0,0 +1,30 @@ +import { Logger } from '@l2beat/backend-tools' +import { setTimeout } from 'timers/promises' + +import { ONE_HOUR_MS } from '../utils' + +export class PriceService { + constructor(private readonly logger: Logger) { + this.logger = logger.for(this) + } + + async getHourlyPrices( + apiId: string, + startHourInclusive: number, + endHourInclusive: number, + ): Promise<{ timestamp: number; price: number }[]> { + const prices: { timestamp: number; price: number }[] = [] + for (let t = startHourInclusive; t <= endHourInclusive; t += ONE_HOUR_MS) { + prices.push({ timestamp: t, price: Math.random() * 1000 }) + } + + await setTimeout(1000) + + this.logger.info('Fetched prices', { + apiId, + since: startHourInclusive, + count: prices.length, + }) + return prices + } +} diff --git a/packages/uif-example/src/utils.ts b/packages/uif-example/src/utils.ts new file mode 100644 index 00000000..55eab79d --- /dev/null +++ b/packages/uif-example/src/utils.ts @@ -0,0 +1,9 @@ +export const ONE_HOUR_MS = 60 * 60 * 1000 + +export function msToHours(timestampOrDate: number | Date): number { + const timestamp = + typeof timestampOrDate === 'number' + ? timestampOrDate + : timestampOrDate.getTime() + return Math.floor(timestamp / ONE_HOUR_MS) +} diff --git a/packages/example/tsconfig.json b/packages/uif-example/tsconfig.json similarity index 100% rename from packages/example/tsconfig.json rename to packages/uif-example/tsconfig.json diff --git a/packages/uif/CHANGELOG.md b/packages/uif/CHANGELOG.md index 1eec5ab6..e1d4126b 100644 --- a/packages/uif/CHANGELOG.md +++ b/packages/uif/CHANGELOG.md @@ -1,5 +1,15 @@ # @l2beat/uif +## 0.3.0 + +### Minor Changes + +- Adds `MultiIndexer` +- Changes the update method `from` parameter to be inclusive as opposed to exclusive (which was the previous behavior) +- Renames `getSafeHeight` to `initialize` +- Renames `BaseIndexer` to `Indexer` +- Removes `SliceIndexer` + ## 0.2.4 ### Patch Changes diff --git a/packages/uif/example/BlockNumberIndexer.example.ts b/packages/uif/example/BlockNumberIndexer.example.ts deleted file mode 100644 index 8fe592f2..00000000 --- a/packages/uif/example/BlockNumberIndexer.example.ts +++ /dev/null @@ -1,176 +0,0 @@ -import { Logger } from '@l2beat/backend-tools' - -import { ChildIndexer, RootIndexer } from '../src' - -interface Hash256 extends String { - _Hash256Brand: string -} - -function Hash256(hash: string): Hash256 { - return hash as unknown as Hash256 -} - -interface UnixTime extends Number { - _UnixTimeBrand: string -} - -function UnixTime(timestamp: number): UnixTime { - return timestamp as unknown as UnixTime -} - -interface BlockFromClient { - parentHash: string - hash: string - number: number - timestamp: number -} - -interface EthereumClient { - getBlock: (block: Hash256 | number) => Promise - getBlockByTimestamp: (timestamp: UnixTime) => Promise - getBlockNumber: () => Promise - onBlock: (handler: (blockNumber: number) => void) => () => void -} - -interface BlockRecord { - number: number - hash: Hash256 - timestamp: number -} - -interface BlockRepository { - findLast: () => Promise - findByNumber: (number: number) => Promise - findByTimestamp: (timestamp: UnixTime) => Promise - addMany: (blocks: BlockRecord[]) => Promise - deleteAfter: (block: number) => Promise -} - -interface IndexerRepository { - getSafeHeight: (indexerId: string) => Promise - setSafeHeight: (indexerId: string, height: number) => Promise -} - -export class ClockIndexer extends RootIndexer { - override async start(): Promise { - await super.start() - setInterval(() => this.requestTick(), 4 * 1000) - } - - async tick(): Promise { - return Promise.resolve(getTimeSeconds()) - } -} - -function getTimeSeconds(): number { - return Math.floor(Date.now() / 1000) -} - -export class BlockDownloader extends ChildIndexer { - private lastKnownNumber = 0 - private reorgedBlocks = [] as BlockRecord[] - private readonly id: string - - constructor( - private readonly ethereumClient: EthereumClient, - private readonly blockRepository: BlockRepository, - private readonly indexerRepository: IndexerRepository, - clockIndexer: ClockIndexer, - logger: Logger, - ) { - super(logger, [clockIndexer]) - this.id = 'BlockDownloader' // this should be unique across all indexers - } - - override async start(): Promise { - await super.start() - this.lastKnownNumber = (await this.blockRepository.findLast())?.number ?? 0 - } - - protected async update( - _fromTimestamp: number, - toTimestamp: number, - ): Promise { - if (this.reorgedBlocks.length > 0) { - // we do not need to check if lastKnown < to because we are sure that - // those blocks are from the past - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - this.lastKnownNumber = this.reorgedBlocks.at(-1)!.number - await this.blockRepository.addMany(this.reorgedBlocks) - this.reorgedBlocks = [] - } - - const tip = await this.ethereumClient.getBlockByTimestamp( - UnixTime(toTimestamp), - ) - if (tip.number <= this.lastKnownNumber) { - return tip.timestamp - } - - return await this.advanceChain(this.lastKnownNumber + 1) - } - - protected async invalidate(to: number): Promise { - await this.blockRepository.deleteAfter(to) - return to - } - - private async advanceChain(blockNumber: number): Promise { - let [block, parent] = await Promise.all([ - this.ethereumClient.getBlock(blockNumber), - this.getKnownBlock(blockNumber - 1), - ]) - if (Hash256(block.parentHash) !== parent.hash) { - const changed = [block] - - let current = blockNumber - while (Hash256(block.parentHash) !== parent.hash) { - current-- - ;[block, parent] = await Promise.all([ - this.ethereumClient.getBlock(Hash256(block.parentHash)), - this.getKnownBlock(current - 1), - ]) - changed.push(block) - } - - this.reorgedBlocks = changed.reverse().map((block) => ({ - number: block.number, - hash: Hash256(block.hash), - timestamp: block.timestamp, - })) - - return parent.timestamp - } - - const record: BlockRecord = { - number: block.number, - hash: Hash256(block.hash), - timestamp: block.timestamp, - } - await this.blockRepository.addMany([record]) - return block.timestamp - } - - protected async setSafeHeight(height: number): Promise { - await this.indexerRepository.setSafeHeight(this.id, height) - } - - getSafeHeight(): Promise { - return this.indexerRepository.getSafeHeight(this.id) - } - - private async getKnownBlock(blockNumber: number): Promise { - const known = await this.blockRepository.findByNumber(blockNumber) - if (known) { - return known - } - const downloaded = await this.ethereumClient.getBlock(blockNumber) - const record: BlockRecord = { - number: downloaded.number, - hash: Hash256(downloaded.hash), - timestamp: downloaded.timestamp, - } - await this.blockRepository.addMany([record]) - return record - } -} diff --git a/packages/uif/example/ConfigurableIndexer.example.ts b/packages/uif/example/ConfigurableIndexer.example.ts deleted file mode 100644 index b1f23243..00000000 --- a/packages/uif/example/ConfigurableIndexer.example.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { Logger } from '@l2beat/backend-tools' - -import { ChildIndexer } from '../src/BaseIndexer' -import { HourlyIndexer } from './HourlyIndexer.example' - -interface IndexerDataRepository { - addData(data: number[]): Promise - removeAfter(number: number): Promise -} - -interface IndexerStateRepository { - getSafeHeight(): Promise - setSafeHeight(height: number): Promise - getConfigHash(): Promise - setConfigHash(hash: string): Promise -} -interface Config { - number: number - getConfigHash(): string -} - -export class ConfigurableIndexer extends ChildIndexer { - constructor( - logger: Logger, - parentIndexer: HourlyIndexer, - private readonly config: Config, - private readonly dataRepository: IndexerDataRepository, - private readonly stateRepository: IndexerStateRepository, - ) { - super(logger, [parentIndexer]) - } - - override async start(): Promise { - const oldConfigHash = await this.stateRepository.getConfigHash() - const newConfigHash = this.config.getConfigHash() - if (oldConfigHash !== newConfigHash) { - await this.stateRepository.setSafeHeight(0) - await this.stateRepository.setConfigHash(newConfigHash) - } - await super.start() - } - - override async update(from: number, to: number): Promise { - const data = [] - for (let i = from + 1; i <= to; i++) { - data.push(i) - } - - await this.dataRepository.addData(data) - return to - } - - override async invalidate(targetHeight: number): Promise { - await this.dataRepository.removeAfter(targetHeight) - return targetHeight - } - - override async getSafeHeight(): Promise { - const height = await this.stateRepository.getSafeHeight() - return height - } - - override async setSafeHeight(height: number): Promise { - return this.stateRepository.setSafeHeight(height) - } -} diff --git a/packages/uif/example/HourlyIndexer.example.ts b/packages/uif/example/HourlyIndexer.example.ts deleted file mode 100644 index eb2fceb0..00000000 --- a/packages/uif/example/HourlyIndexer.example.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { RootIndexer } from '../src/BaseIndexer' - -const MS_IN_HOUR = 60 * 60 * 1000 - -export class HourlyIndexer extends RootIndexer { - override async start(): Promise { - await super.start() - setInterval(() => this.requestTick(), MS_IN_HOUR) - } - - tick(): Promise { - const time = getLastFullHourTimestampSeconds() - return Promise.resolve(time) - } -} - -function getLastFullHourTimestampSeconds(): number { - const now = Date.now() - const lastFullHour = now - (now % MS_IN_HOUR) - return Math.floor(lastFullHour / 1000) -} diff --git a/packages/uif/package.json b/packages/uif/package.json index 2c460a71..8f661697 100644 --- a/packages/uif/package.json +++ b/packages/uif/package.json @@ -1,7 +1,7 @@ { "name": "@l2beat/uif", "description": "Universal Indexer Framework.", - "version": "0.2.4", + "version": "0.3.0", "license": "MIT", "repository": "https://github.com/l2beat/tools", "bugs": { diff --git a/packages/uif/src/BaseIndexer.ts b/packages/uif/src/BaseIndexer.ts deleted file mode 100644 index f8bf88d0..00000000 --- a/packages/uif/src/BaseIndexer.ts +++ /dev/null @@ -1,338 +0,0 @@ -import { assert } from 'node:console' - -import { Logger } from '@l2beat/backend-tools' - -import { assertUnreachable } from './assertUnreachable' -import { Indexer } from './Indexer' -import { getInitialState } from './reducer/getInitialState' -import { indexerReducer } from './reducer/indexerReducer' -import { IndexerAction } from './reducer/types/IndexerAction' -import { - InvalidateEffect, - NotifyReadyEffect, - SetSafeHeightEffect, - UpdateEffect, -} from './reducer/types/IndexerEffect' -import { IndexerState } from './reducer/types/IndexerState' -import { Retries, RetryStrategy } from './Retries' - -export abstract class BaseIndexer implements Indexer { - private readonly children: Indexer[] = [] - - /** - * This can be overridden to provide a custom retry strategy. It will be - * used for all indexers in the app that don't specify their own strategy. - * @returns A default retry strategy that will be used for all indexers in the app - */ - static GET_DEFAULT_RETRY_STRATEGY: () => RetryStrategy = () => - Retries.exponentialBackOff({ - initialTimeoutMs: 1000, - maxAttempts: 10, - maxTimeoutMs: 60 * 1000, - }) - - /** - * Should read the height from the database. It must return a height, so - * if database is empty a fallback value has to be chosen. - */ - abstract getSafeHeight(): Promise - - /** - * Should write the height to the database. The height given is the most - * pessimistic value and the indexer is expected to actually operate at a - * higher height in runtime. - */ - protected abstract setSafeHeight(height: number): Promise - - /** - * To be used in ChildIndexer. - * - * @param from - current height of the indexer, exclusive - * @param to - inclusive - */ - // TODO: do we need to pass the current height? - protected abstract update(from: number, to: number): Promise - - /** - * Only to be used in RootIndexer. It provides a way to start the height - * update process. - */ - protected abstract tick(): Promise - - /** - * @param targetHeight - every value > `targetHeight` is invalid, but `targetHeight` itself is valid - */ - protected abstract invalidate(targetHeight: number): Promise - - private state: IndexerState - private started = false - private readonly tickRetryStrategy: RetryStrategy - private readonly updateRetryStrategy: RetryStrategy - private readonly invalidateRetryStrategy: RetryStrategy - - constructor( - protected logger: Logger, - public readonly parents: Indexer[], - opts?: { - tickRetryStrategy?: RetryStrategy - updateRetryStrategy?: RetryStrategy - invalidateRetryStrategy?: RetryStrategy - }, - ) { - this.logger = this.logger.for(this) - this.state = getInitialState(parents.length) - this.parents.forEach((parent) => { - this.logger.debug('Subscribing to parent', { - parent: parent.constructor.name, - }) - parent.subscribe(this) - }) - - this.tickRetryStrategy = - opts?.tickRetryStrategy ?? BaseIndexer.GET_DEFAULT_RETRY_STRATEGY() - this.updateRetryStrategy = - opts?.updateRetryStrategy ?? BaseIndexer.GET_DEFAULT_RETRY_STRATEGY() - this.invalidateRetryStrategy = - opts?.invalidateRetryStrategy ?? BaseIndexer.GET_DEFAULT_RETRY_STRATEGY() - } - - async start(): Promise { - assert(!this.started, 'Indexer already started') - this.started = true - const height = await this.getSafeHeight() - this.dispatch({ - type: 'Initialized', - safeHeight: height, - childCount: this.children.length, - }) - } - - subscribe(child: Indexer): void { - assert(!this.started, 'Indexer already started') - this.logger.debug('Child subscribed', { child: child.constructor.name }) - this.children.push(child) - } - - notifyReady(child: Indexer): void { - this.logger.debug('Someone is ready', { child: child.constructor.name }) - const index = this.children.indexOf(child) - assert(index !== -1, 'Received ready from unknown child') - this.dispatch({ type: 'ChildReady', index }) - } - - notifyUpdate(parent: Indexer, to: number): void { - this.logger.debug('Someone has updated', { - parent: parent.constructor.name, - }) - const index = this.parents.indexOf(parent) - assert(index !== -1, 'Received update from unknown parent') - this.dispatch({ type: 'ParentUpdated', index, safeHeight: to }) - } - - getState(): IndexerState { - return this.state - } - - private dispatch(action: IndexerAction): void { - const [newState, effects] = indexerReducer(this.state, action) - this.state = newState - this.logger.debug('Action', { action }) - this.logger.trace('State', { state: newState, effects }) - - // TODO: check if this doesn't result in stack overflow - effects.forEach((effect) => { - switch (effect.type) { - case 'Update': - return void this.executeUpdate(effect) - case 'Invalidate': - return void this.executeInvalidate(effect) - case 'SetSafeHeight': - return void this.executeSetSafeHeight(effect) - case 'NotifyReady': - return this.executeNotifyReady(effect) - case 'Tick': - return void this.executeTick() - case 'ScheduleRetryUpdate': - return this.executeScheduleRetryUpdate() - case 'ScheduleRetryInvalidate': - return this.executeScheduleRetryInvalidate() - case 'ScheduleRetryTick': - return this.executeScheduleRetryTick() - default: - return assertUnreachable(effect) - } - }) - } - - // #region Child methods - - private async executeUpdate(effect: UpdateEffect): Promise { - // TODO: maybe from should be inclusive? - const from = this.state.height - this.logger.info('Updating', { from, to: effect.targetHeight }) - try { - const to = await this.update(from, effect.targetHeight) - if (to > effect.targetHeight) { - this.logger.critical('Update returned invalid height', { - returned: to, - max: effect.targetHeight, - }) - this.dispatch({ type: 'UpdateFailed', fatal: true }) - } else { - this.dispatch({ type: 'UpdateSucceeded', from, targetHeight: to }) - this.updateRetryStrategy.clear() - } - } catch (e) { - this.updateRetryStrategy.markAttempt() - const fatal = !this.updateRetryStrategy.shouldRetry() - if (fatal) { - this.logger.critical('Update failed', e) - } else { - this.logger.error('Update failed', e) - } - this.dispatch({ type: 'UpdateFailed', fatal }) - } - } - - private executeScheduleRetryUpdate(): void { - const timeoutMs = this.updateRetryStrategy.timeoutMs() - this.logger.debug('Scheduling retry update', { timeoutMs }) - setTimeout(() => { - this.dispatch({ type: 'RetryUpdate' }) - }, timeoutMs) - } - - private async executeInvalidate(effect: InvalidateEffect): Promise { - this.logger.info('Invalidating', { to: effect.targetHeight }) - try { - const to = await this.invalidate(effect.targetHeight) - this.dispatch({ - type: 'InvalidateSucceeded', - targetHeight: to, - }) - this.invalidateRetryStrategy.clear() - } catch (e) { - this.invalidateRetryStrategy.markAttempt() - const fatal = !this.invalidateRetryStrategy.shouldRetry() - if (fatal) { - this.logger.critical('Invalidate failed', e) - } else { - this.logger.error('Invalidate failed', e) - } - this.dispatch({ type: 'InvalidateFailed', fatal }) - } - } - - private executeScheduleRetryInvalidate(): void { - const timeoutMs = this.invalidateRetryStrategy.timeoutMs() - this.logger.debug('Scheduling retry invalidate', { timeoutMs }) - setTimeout(() => { - this.dispatch({ type: 'RetryInvalidate' }) - }, timeoutMs) - } - - private executeNotifyReady(effect: NotifyReadyEffect): void { - this.parents.forEach((parent, index) => { - if (effect.parentIndices.includes(index)) { - parent.notifyReady(this) - } - }) - } - - // #endregion - // #region Root methods - - private async executeTick(): Promise { - this.logger.debug('Ticking') - try { - const safeHeight = await this.tick() - this.dispatch({ type: 'TickSucceeded', safeHeight }) - this.tickRetryStrategy.clear() - } catch (e) { - this.tickRetryStrategy.markAttempt() - const fatal = !this.tickRetryStrategy.shouldRetry() - if (fatal) { - this.logger.critical('Tick failed', e) - } else { - this.logger.error('Tick failed', e) - } - this.dispatch({ type: 'TickFailed', fatal }) - } - } - - private executeScheduleRetryTick(): void { - const timeoutMs = this.tickRetryStrategy.timeoutMs() - this.logger.debug('Scheduling retry tick', { timeoutMs }) - setTimeout(() => { - this.dispatch({ type: 'RetryTick' }) - }, timeoutMs) - } - - /** - * Requests the tick function to be called. Repeated calls will result in - * only one tick. Only when the tick is finished, the next tick will be - * scheduled. - */ - protected requestTick(): void { - this.logger.trace('Requesting tick') - this.dispatch({ type: 'RequestTick' }) - } - - // #endregion - // #region Common methods - - private async executeSetSafeHeight( - effect: SetSafeHeightEffect, - ): Promise { - this.logger.info('Setting safe height', { safeHeight: effect.safeHeight }) - this.children.forEach((child) => - child.notifyUpdate(this, effect.safeHeight), - ) - await this.setSafeHeight(effect.safeHeight) - } - - // #endregion -} - -export abstract class RootIndexer extends BaseIndexer { - constructor(logger: Logger, opts?: { tickRetryStrategy?: RetryStrategy }) { - super(logger, [], opts) - } - - // eslint-disable-next-line @typescript-eslint/require-await - override async update(): Promise { - throw new Error('RootIndexer cannot update') - } - - // eslint-disable-next-line @typescript-eslint/require-await - override async invalidate(): Promise { - throw new Error('RootIndexer cannot invalidate') - } - - override async getSafeHeight(): Promise { - return this.tick() - } - - override async setSafeHeight(): Promise { - return Promise.resolve() - } -} - -export abstract class ChildIndexer extends BaseIndexer { - // eslint-disable-next-line @typescript-eslint/no-useless-constructor - constructor( - logger: Logger, - parents: Indexer[], - opts?: { - updateRetryStrategy?: RetryStrategy - invalidateRetryStrategy?: RetryStrategy - }, - ) { - super(logger, parents, opts) - } - - // eslint-disable-next-line @typescript-eslint/require-await - override async tick(): Promise { - throw new Error('ChildIndexer cannot tick') - } -} diff --git a/packages/uif/src/BaseIndexer.test.ts b/packages/uif/src/Indexer.test.ts similarity index 89% rename from packages/uif/src/BaseIndexer.test.ts rename to packages/uif/src/Indexer.test.ts index 585db584..c6267e18 100644 --- a/packages/uif/src/BaseIndexer.test.ts +++ b/packages/uif/src/Indexer.test.ts @@ -2,11 +2,13 @@ import { Logger } from '@l2beat/backend-tools' import { install } from '@sinonjs/fake-timers' import { expect, mockFn } from 'earl' -import { BaseIndexer, ChildIndexer, RootIndexer } from './BaseIndexer' +import { Indexer } from './Indexer' +import { ChildIndexer } from './indexers/ChildIndexer' +import { RootIndexer } from './indexers/RootIndexer' import { IndexerAction } from './reducer/types/IndexerAction' import { RetryStrategy } from './Retries' -describe(BaseIndexer.name, () => { +describe(Indexer.name, () => { describe('correctly informs about updates', () => { it('first invalidate then parent update', async () => { const parent = new TestRootIndexer(0) @@ -21,7 +23,7 @@ describe(BaseIndexer.name, () => { await child.finishUpdate(1) - expect(await child.getSafeHeight()).toEqual(1) + expect(await child.initialize()).toEqual(1) }) it('first parent update then invalidate', async () => { @@ -37,7 +39,7 @@ describe(BaseIndexer.name, () => { await child.finishUpdate(1) - expect(await child.getSafeHeight()).toEqual(1) + expect(await child.initialize()).toEqual(1) }) }) @@ -212,6 +214,23 @@ describe(BaseIndexer.name, () => { clock.uninstall() }) }) + + it('calls update with correct heights', async () => { + const parent = new TestRootIndexer(100) + const child = new TestChildIndexer([parent], 100) + + await parent.start() + await child.start() + await child.finishInvalidate(100) + + await parent.doTick(200) + await parent.finishTick(200) + + expect(child.updateFrom).toEqual(101) // inclusive + expect(child.updateTo).toEqual(200) // inclusive + + await child.finishUpdate(200) + }) }) export async function waitUntil(predicate: () => boolean): Promise { @@ -233,7 +252,7 @@ export class TestRootIndexer extends RootIndexer { ticking = false constructor( - private safeHeight: number, + private testSafeHeight: number, name?: string, retryStrategy?: { tickRetryStrategy?: RetryStrategy }, ) { @@ -248,7 +267,7 @@ export class TestRootIndexer extends RootIndexer { async doTick(height: number): Promise { await waitUntil(() => this.getState().status === 'idle') - this.safeHeight = height + this.testSafeHeight = height const counter = this.dispatchCounter this.requestTick() await waitUntil(() => this.dispatchCounter > counter) @@ -265,7 +284,7 @@ export class TestRootIndexer extends RootIndexer { await waitUntil(() => this.dispatchCounter > counter) } - override tick(): Promise { + override async tick(): Promise { this.ticking = true return new Promise((resolve, reject) => { @@ -276,11 +295,11 @@ export class TestRootIndexer extends RootIndexer { }) } - override async getSafeHeight(): Promise { + override async initialize(): Promise { const promise = this.tick() - this.resolveTick(this.safeHeight) + this.resolveTick(this.testSafeHeight) await promise - return this.safeHeight + return this.testSafeHeight } } @@ -322,8 +341,8 @@ class TestChildIndexer extends ChildIndexer { public invalidateTo = 0 constructor( - parents: BaseIndexer[], - private safeHeight: number, + parents: Indexer[], + private testSafeHeight: number, name?: string, retryStrategy?: { invalidateRetryStrategy?: RetryStrategy @@ -339,12 +358,12 @@ class TestChildIndexer extends ChildIndexer { }) } - override getSafeHeight(): Promise { - return Promise.resolve(this.safeHeight) + override initialize(): Promise { + return Promise.resolve(this.testSafeHeight) } override setSafeHeight(safeHeight: number): Promise { - this.safeHeight = safeHeight + this.testSafeHeight = safeHeight return Promise.resolve() } diff --git a/packages/uif/src/Indexer.ts b/packages/uif/src/Indexer.ts index 6b23c216..32c9069e 100644 --- a/packages/uif/src/Indexer.ts +++ b/packages/uif/src/Indexer.ts @@ -1,11 +1,346 @@ -export interface UpdateEvent { - type: 'update' - height: number +import { assert } from 'node:console' + +import { Logger } from '@l2beat/backend-tools' + +import { assertUnreachable } from './assertUnreachable' +import { getInitialState } from './reducer/getInitialState' +import { indexerReducer } from './reducer/indexerReducer' +import { IndexerAction } from './reducer/types/IndexerAction' +import { + InvalidateEffect, + NotifyReadyEffect, + SetSafeHeightEffect, + UpdateEffect, +} from './reducer/types/IndexerEffect' +import { IndexerState } from './reducer/types/IndexerState' +import { Retries, RetryStrategy } from './Retries' + +export interface IndexerOptions { + tickRetryStrategy?: RetryStrategy + updateRetryStrategy?: RetryStrategy + invalidateRetryStrategy?: RetryStrategy } -export interface Indexer { - subscribe(child: Indexer): void - notifyReady(child: Indexer): void - notifyUpdate(parent: Indexer, safeHeight: number): void - start(): Promise +export abstract class Indexer { + private readonly children: Indexer[] = [] + + /** + * This can be overridden to provide a custom retry strategy. It will be + * used for all indexers that don't specify their own strategy. + * @returns A default retry strategy that will be used for all indexers + */ + static GET_DEFAULT_RETRY_STRATEGY: () => RetryStrategy = () => + Retries.exponentialBackOff({ + initialTimeoutMs: 1000, + maxAttempts: 10, + maxTimeoutMs: 60 * 1000, + }) + + /** + * Initializes the indexer. It should return a height that the indexer has + * synced up to. If the indexer has not synced any data, it should return + * `minHeight - 1`. For root indexers it should return the initial target + * height for the entire system. + * + * This method is expected to read the height that was saved previously with + * `setSafeHeight`. It shouldn't call `setSafeHeight` itself. + * + * For root indexers if `setSafeHeight` is implemented it should return the + * height that was saved previously. If not it can `return this.tick()`. + * This method should also schedule a process to request ticks. For example + * with `setInterval(() => this.requestTick(), 1000)`. + * + * @returns The height that the indexer has synced up to or the target height + * for the entire system if this is a root indexer. + */ + abstract initialize(): Promise + + /** + * Saves the height (most likely to a database). The height given is the + * smallest height from all parents and what the indexer itself synced to + * previously. + * + * When `initialize` is called it is expected that it will read the same + * height that was saved here. + * + * Optional in root indexers. + */ + abstract setSafeHeight(height: number): Promise + + /** + * Implements the main data fetching process. It is up to the indexer to + * decide how much data to fetch. For example given `.update(100, 200)`, the + * indexer can only fetch data up to 110 and return 110. The next time this + * method will be called with `.update(111, 200)`. + * + * @param from The height for which the indexer should start syncing data. + * This value is inclusive. + * + * @param to The height at which the indexer should end syncing data. This + * value is also inclusive so the indexer should eventually sync data for this + * height. + * + * @returns The height that the indexer has synced up to. Returning + * `from` means that the indexer has synced a single data point. Returning + * a value greater than `from` means that the indexer has synced up + * to that height. Returning a value less than `from` will trigger + * invalidation down to the returned value. Returning a value greater than + * `to` is not permitted. + */ + abstract update(from: number, to: number): Promise + + /** + * Responsible for invalidating data that was synced previously. It is + * possible that no data was synced and this method is still called. + * + * Invalidation can, but doesn't have to remove data from the database. If + * you only want to rely on the safe height, you can just return the target + * height and the system will take care of the rest. + * + * This method doesn't have to invalidate all data. If you want to do it in + * steps, you can return a height that is larger than the target height. + * + * @param targetHeight The height that the indexer should invalidate down to. + * + * @returns The height that the indexer has invalidated down to. Returning + * `targetHeight` means that the indexer has invalidated all the required + * data. Returning a value greater than `targetHeight` means that the indexer + * has invalidated down to that height. + */ + abstract invalidate(targetHeight: number): Promise + + /** + * This method is responsible for providing the target height for the entire + * system. Some candidates for this are: the current time or the latest block + * number. + */ + abstract tick(): Promise + + private state: IndexerState + private started = false + private readonly tickRetryStrategy: RetryStrategy + private readonly updateRetryStrategy: RetryStrategy + private readonly invalidateRetryStrategy: RetryStrategy + + constructor( + protected logger: Logger, + public readonly parents: Indexer[], + options?: IndexerOptions, + ) { + this.logger = this.logger.for(this) + this.state = getInitialState(parents.length) + this.parents.forEach((parent) => { + this.logger.debug('Subscribing to parent', { + parent: parent.constructor.name, + }) + parent.subscribe(this) + }) + + this.tickRetryStrategy = + options?.tickRetryStrategy ?? Indexer.GET_DEFAULT_RETRY_STRATEGY() + this.updateRetryStrategy = + options?.updateRetryStrategy ?? Indexer.GET_DEFAULT_RETRY_STRATEGY() + this.invalidateRetryStrategy = + options?.invalidateRetryStrategy ?? Indexer.GET_DEFAULT_RETRY_STRATEGY() + } + + get safeHeight(): number { + return this.state.safeHeight + } + + async start(): Promise { + assert(!this.started, 'Indexer already started') + this.started = true + const height = await this.initialize() + this.dispatch({ + type: 'Initialized', + safeHeight: height, + childCount: this.children.length, + }) + } + + subscribe(child: Indexer): void { + assert(!this.started, 'Indexer already started') + this.logger.debug('Child subscribed', { child: child.constructor.name }) + this.children.push(child) + } + + notifyReady(child: Indexer): void { + this.logger.debug('Someone is ready', { child: child.constructor.name }) + const index = this.children.indexOf(child) + assert(index !== -1, 'Received ready from unknown child') + this.dispatch({ type: 'ChildReady', index }) + } + + notifyUpdate(parent: Indexer, safeHeight: number): void { + this.logger.debug('Someone has updated', { + parent: parent.constructor.name, + }) + const index = this.parents.indexOf(parent) + assert(index !== -1, 'Received update from unknown parent') + this.dispatch({ type: 'ParentUpdated', index, safeHeight }) + } + + getState(): IndexerState { + return this.state + } + + private dispatch(action: IndexerAction): void { + const [newState, effects] = indexerReducer(this.state, action) + this.state = newState + this.logger.debug('Action', { action }) + this.logger.trace('State', { state: newState, effects }) + + // TODO: check if this doesn't result in stack overflow + effects.forEach((effect) => { + switch (effect.type) { + case 'Update': + return void this.executeUpdate(effect) + case 'Invalidate': + return void this.executeInvalidate(effect) + case 'SetSafeHeight': + return void this.executeSetSafeHeight(effect) + case 'NotifyReady': + return this.executeNotifyReady(effect) + case 'Tick': + return void this.executeTick() + case 'ScheduleRetryUpdate': + return this.executeScheduleRetryUpdate() + case 'ScheduleRetryInvalidate': + return this.executeScheduleRetryInvalidate() + case 'ScheduleRetryTick': + return this.executeScheduleRetryTick() + default: + return assertUnreachable(effect) + } + }) + } + + // #region Child methods + + private async executeUpdate(effect: UpdateEffect): Promise { + const from = this.state.height + 1 + this.logger.info('Updating', { from, to: effect.targetHeight }) + try { + const newHeight = await this.update(from, effect.targetHeight) + if (newHeight > effect.targetHeight) { + this.logger.critical('Update returned invalid height', { + newHeight, + max: effect.targetHeight, + }) + this.dispatch({ type: 'UpdateFailed', fatal: true }) + } else { + this.dispatch({ type: 'UpdateSucceeded', from, newHeight }) + this.updateRetryStrategy.clear() + } + } catch (e) { + this.updateRetryStrategy.markAttempt() + const fatal = !this.updateRetryStrategy.shouldRetry() + if (fatal) { + this.logger.critical('Update failed', e) + } else { + this.logger.error('Update failed', e) + } + this.dispatch({ type: 'UpdateFailed', fatal }) + } + } + + private executeScheduleRetryUpdate(): void { + const timeoutMs = this.updateRetryStrategy.timeoutMs() + this.logger.debug('Scheduling retry update', { timeoutMs }) + setTimeout(() => { + this.dispatch({ type: 'RetryUpdate' }) + }, timeoutMs) + } + + private async executeInvalidate(effect: InvalidateEffect): Promise { + this.logger.info('Invalidating', { to: effect.targetHeight }) + try { + const to = await this.invalidate(effect.targetHeight) + this.dispatch({ + type: 'InvalidateSucceeded', + targetHeight: to, + }) + this.invalidateRetryStrategy.clear() + } catch (e) { + this.invalidateRetryStrategy.markAttempt() + const fatal = !this.invalidateRetryStrategy.shouldRetry() + if (fatal) { + this.logger.critical('Invalidate failed', e) + } else { + this.logger.error('Invalidate failed', e) + } + this.dispatch({ type: 'InvalidateFailed', fatal }) + } + } + + private executeScheduleRetryInvalidate(): void { + const timeoutMs = this.invalidateRetryStrategy.timeoutMs() + this.logger.debug('Scheduling retry invalidate', { timeoutMs }) + setTimeout(() => { + this.dispatch({ type: 'RetryInvalidate' }) + }, timeoutMs) + } + + private executeNotifyReady(effect: NotifyReadyEffect): void { + this.parents.forEach((parent, index) => { + if (effect.parentIndices.includes(index)) { + parent.notifyReady(this) + } + }) + } + + // #endregion + // #region Root methods + + private async executeTick(): Promise { + this.logger.debug('Ticking') + try { + const safeHeight = await this.tick() + this.dispatch({ type: 'TickSucceeded', safeHeight }) + this.tickRetryStrategy.clear() + } catch (e) { + this.tickRetryStrategy.markAttempt() + const fatal = !this.tickRetryStrategy.shouldRetry() + if (fatal) { + this.logger.critical('Tick failed', e) + } else { + this.logger.error('Tick failed', e) + } + this.dispatch({ type: 'TickFailed', fatal }) + } + } + + private executeScheduleRetryTick(): void { + const timeoutMs = this.tickRetryStrategy.timeoutMs() + this.logger.debug('Scheduling retry tick', { timeoutMs }) + setTimeout(() => { + this.dispatch({ type: 'RetryTick' }) + }, timeoutMs) + } + + /** + * Requests the tick function to be called. Repeated calls will result in + * only one tick. Only when the tick is finished, the next tick will be + * scheduled. + */ + protected requestTick(): void { + this.logger.trace('Requesting tick') + this.dispatch({ type: 'RequestTick' }) + } + + // #endregion + // #region Common methods + + private async executeSetSafeHeight( + effect: SetSafeHeightEffect, + ): Promise { + this.logger.info('Setting safe height', { safeHeight: effect.safeHeight }) + this.children.forEach((child) => + child.notifyUpdate(this, effect.safeHeight), + ) + await this.setSafeHeight(effect.safeHeight) + } + + // #endregion } diff --git a/packages/uif/src/SliceIndexer.test.ts b/packages/uif/src/SliceIndexer.test.ts deleted file mode 100644 index 5273e6e3..00000000 --- a/packages/uif/src/SliceIndexer.test.ts +++ /dev/null @@ -1,344 +0,0 @@ -import { Logger } from '@l2beat/backend-tools' -import { expect, mockFn } from 'earl' - -import { BaseIndexer } from './BaseIndexer' -import { TestRootIndexer, waitUntil } from './BaseIndexer.test' -import { IndexerAction } from './reducer/types/IndexerAction' -import { RetryStrategy } from './Retries' -import { - diffSlices, - SliceHash, - SliceIndexer, - SliceState, - SliceUpdate, -} from './SliceIndexer' - -describe(SliceIndexer.name, () => { - it('updates multiple slices', async () => { - const repository = testRepositoryFactory() - - const rootIndexer = new TestRootIndexer(0) - const sliceIndexer = new TestSliceIndexer( - ['token-a', 'token-b', 'token-c'], - repository, - [rootIndexer], - ) - - await rootIndexer.start() - await sliceIndexer.start() - - await sliceIndexer.finishInvalidate(0) - await rootIndexer.doTick(1) - await rootIndexer.finishTick(1) - - expect(repository.getSliceData('token-a').get(1)).toEqual(2) - expect(repository.getSliceData('token-b').get(1)).toEqual(2) - expect(repository.getSliceData('token-c').get(1)).toEqual(2) - - await rootIndexer.doTick(2) - await rootIndexer.finishTick(2) - - expect(repository.getSliceData('token-a').get(2)).toEqual(4) - expect(repository.getSliceData('token-b').get(2)).toEqual(4) - expect(repository.getSliceData('token-c').get(2)).toEqual(4) - }) - it('syncs new token after restart', async () => { - const repository = testRepositoryFactory() - - const rootIndexer = new TestRootIndexer(0) - const sliceIndexer = new TestSliceIndexer( - ['token-a', 'token-b', 'token-c'], - repository, - [rootIndexer], - ) - - await rootIndexer.start() - await sliceIndexer.start() - - await sliceIndexer.finishInvalidate(0) - await rootIndexer.doTick(1) - await rootIndexer.finishTick(1) - - expect(repository.getSliceData('token-a').get(1)).toEqual(2) - expect(repository.getSliceData('token-b').get(1)).toEqual(2) - expect(repository.getSliceData('token-c').get(1)).toEqual(2) - - // creating new instances to simulate restart - const newRootIndexer = new TestRootIndexer(1) - const newSliceIndexer = new TestSliceIndexer( - ['token-a', 'token-b', 'token-c', 'token-d'], - repository, - [newRootIndexer], - ) - - await newRootIndexer.start() - await newSliceIndexer.start() - - await newSliceIndexer.finishInvalidate(0) - - expect(repository.getSliceData('token-d').get(1)).toEqual(2) - }) - it('drops old token after restart', async () => { - const repository = testRepositoryFactory() - - const rootIndexer = new TestRootIndexer(0) - const sliceIndexer = new TestSliceIndexer( - ['token-a', 'token-b', 'token-c'], - repository, - [rootIndexer], - ) - - await rootIndexer.start() - await sliceIndexer.start() - - await sliceIndexer.finishInvalidate(0) - await rootIndexer.doTick(1) - await rootIndexer.finishTick(1) - - expect(repository.getSliceData('token-a').get(1)).toEqual(2) - expect(repository.getSliceData('token-b').get(1)).toEqual(2) - expect(repository.getSliceData('token-c').get(1)).toEqual(2) - - // creating new instances to simulate restart - const newRootIndexer = new TestRootIndexer(1) - const newSliceIndexer = new TestSliceIndexer( - ['token-a', 'token-b'], - repository, - [newRootIndexer], - ) - - await newRootIndexer.start() - await newSliceIndexer.start() - - await newSliceIndexer.finishInvalidate(0) - - expect(repository.removeSlices).toHaveBeenCalledWith(['token-c']) - expect(repository.getSliceData('token-c').get(1)).toBeNullish() - }) -}) -describe(diffSlices.name, () => { - describe('height increase', () => { - it('marks token-a to update', () => { - const expectedSlices: SliceHash[] = ['token-a', 'token-b', 'token-c'] - const actualSlices: SliceState[] = [ - { sliceHash: 'token-a', height: 100 }, - { sliceHash: 'token-b', height: 200 }, - { sliceHash: 'token-c', height: 300 }, - ] - const from = 100 - const to = 200 - - const { toRemove, toUpdate } = diffSlices( - expectedSlices, - actualSlices, - from, - to, - ) - - expect(toRemove).toEqual([]) - expect(toUpdate).toEqual([{ sliceHash: 'token-a', from, to }]) - }) - }) - describe('height decrease', () => { - it('marks token-a to update', () => { - const expectedSlices: SliceHash[] = ['token-a', 'token-b', 'token-c'] - const actualSlices: SliceState[] = [ - { sliceHash: 'token-a', height: 100 }, - { sliceHash: 'token-b', height: 200 }, - { sliceHash: 'token-c', height: 300 }, - ] - const from = 300 - const to = 200 - - const { toUpdate, toRemove } = diffSlices( - expectedSlices, - actualSlices, - from, - to, - ) - - expect(toRemove).toEqual([]) - expect(toUpdate).toEqual([{ sliceHash: 'token-a', from, to }]) - }) - }) - - describe('configuration change', () => { - it('marks token-b to removal from slices state', () => { - const expectedSlices: SliceHash[] = ['token-a', 'token-c'] - const actualSlices: SliceState[] = [ - { sliceHash: 'token-a', height: 100 }, - { sliceHash: 'token-b', height: 200 }, - { sliceHash: 'token-c', height: 300 }, - ] - const from = 100 - const to = 200 - - const { toUpdate, toRemove } = diffSlices( - expectedSlices, - actualSlices, - from, - to, - ) - - expect(toRemove).toEqual(['token-b']) - expect(toUpdate).toEqual([{ sliceHash: 'token-a', from, to }]) - }) - - it('marks token-d to addition to slices state', () => { - const expectedSlices: SliceHash[] = [ - 'token-a', - 'token-b', - 'token-c', - 'token-d', - ] - - const actualSlices: SliceState[] = [ - { sliceHash: 'token-a', height: 200 }, - { sliceHash: 'token-b', height: 200 }, - { sliceHash: 'token-c', height: 200 }, - ] - - const from = 100 - const to = 200 - - const { toUpdate, toRemove } = diffSlices( - expectedSlices, - actualSlices, - from, - to, - ) - - expect(toRemove).toEqual([]) - expect(toUpdate).toEqual([{ sliceHash: 'token-d', from: 100, to: 200 }]) - }) - }) -}) - -function testRepositoryFactory() { - const sliceHeights = new Map() - const sliceData = new Map>() - let safeHeight = 0 - - return { - getSliceHeights: mockFn(() => sliceHeights), - getSliceData: mockFn( - (hash: string) => sliceData.get(hash) ?? new Map(), - ), - removeSlices: mockFn((hashes: string[]) => { - for (const hash of hashes) { - sliceHeights.delete(hash) - sliceData.delete(hash) - } - }), - setSliceHeight: mockFn((hash: string, height: number) => { - sliceHeights.set(hash, height) - }), - setSliceData: mockFn((hash: string, data: Map) => { - sliceData.set(hash, data) - }), - getSafeHeight: mockFn(() => { - return safeHeight - }), - setSafeHeight: mockFn((height: number) => { - safeHeight = height - }), - } -} - -class TestSliceIndexer extends SliceIndexer { - public resolveInvalidate: (to: number) => void = () => {} - public rejectInvalidate: (error: unknown) => void = () => {} - - public dispatchCounter = 0 - - async finishInvalidate(result: number | Error): Promise { - await waitUntil(() => this.invalidating) - const counter = this.dispatchCounter - if (typeof result === 'number') { - this.resolveInvalidate(result) - } else { - this.rejectInvalidate(result) - } - await waitUntil(() => this.dispatchCounter > counter) - } - - public invalidating = false - public invalidateTo = 0 - - constructor( - private readonly slices: string[], - private readonly repository: ReturnType, - parents: BaseIndexer[], - name?: string, - retryStrategy?: { - invalidateRetryStrategy?: RetryStrategy - updateRetryStrategy?: RetryStrategy - }, - ) { - super(Logger.SILENT.tag(name), parents, retryStrategy ?? {}) - - const oldDispatch = Reflect.get(this, 'dispatch') - Reflect.set(this, 'dispatch', (action: IndexerAction) => { - oldDispatch.call(this, action) - this.dispatchCounter++ - }) - } - - override getMainSafeHeight(): Promise { - return Promise.resolve(this.repository.getSafeHeight()) - } - - override setMainSafeHeight(safeHeight: number): Promise { - this.repository.setSafeHeight(safeHeight) - return Promise.resolve() - } - - override async invalidate(to: number): Promise { - this.invalidating = true - this.invalidateTo = to - return new Promise((resolve, reject) => { - this.resolveInvalidate = resolve - this.rejectInvalidate = reject - }).finally(() => { - this.invalidating = false - }) - } - - getExpectedSlices(): string[] { - return this.slices - } - - getSliceStates(): Promise { - const sliceHeights = this.repository.getSliceHeights() - const states = [...sliceHeights.entries()].map( - ([slice, height]): SliceState => ({ - sliceHash: slice, - height, - }), - ) - return Promise.resolve(states) - } - - removeSlices(hashes: string[]): Promise { - this.repository.removeSlices(hashes) - return Promise.resolve() - } - - async updateSlices(updates: SliceUpdate[]): Promise { - let minHeight = Infinity - for (const update of updates) { - const sliceData = this.repository.getSliceData(update.sliceHash) - - for (let i = update.from; i <= update.to; i++) { - sliceData.set(i, i * 2) - } - if (update.to < minHeight) { - minHeight = update.to - } - - this.repository.setSliceData(update.sliceHash, sliceData) - this.repository.setSliceHeight(update.sliceHash, update.to) - } - return Promise.resolve(minHeight) - } -} diff --git a/packages/uif/src/SliceIndexer.ts b/packages/uif/src/SliceIndexer.ts deleted file mode 100644 index 7992f38e..00000000 --- a/packages/uif/src/SliceIndexer.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { ChildIndexer } from './BaseIndexer' - -export type SliceHash = string - -export interface SliceState { - sliceHash: SliceHash - height: number -} - -export interface SliceUpdate { - sliceHash: SliceHash - from: number - to: number -} - -export abstract class SliceIndexer extends ChildIndexer { - override async update(from: number, to: number): Promise { - const sliceStates = await this.getSliceStates() - const { toUpdate, toRemove } = diffSlices( - this.getExpectedSlices(), - sliceStates, - from, - to, - ) - - this.logger.info('Update', { - amountToUpdate: toUpdate.length, - amountToRemove: toRemove.length, - }) - - if (toRemove.length > 0) { - this.logger.debug('Removing slices', { toRemove }) - await this.removeSlices(toRemove) - this.logger.debug('Removed slices') - } - - if (toUpdate.length > 0) { - this.logger.debug('Updating slices', { toUpdate }) - const newHeight = await this.updateSlices(toUpdate) - this.logger.debug('Updated slices', { newHeight }) - return newHeight - } - - return to - } - - override async getSafeHeight(): Promise { - const sliceStates = await this.getSliceStates() - const mainSafeHeight = await this.getMainSafeHeight() - return Math.min(...sliceStates.map((s) => s.height), mainSafeHeight) - } - - override async setSafeHeight(height: number): Promise { - await this.setMainSafeHeight(height) - } - - /** - * @returns Height of the indexer. Should be saved in persistent storage. - */ - abstract getMainSafeHeight(): Promise - - /** - * @param height Height of the indexer. Should be saved in persistent storage. - */ - abstract setMainSafeHeight(height: number): Promise - - /** - * @returns Slices derived from config - */ - abstract getExpectedSlices(): SliceHash[] - - /** - * @returns State for Slices stored in the database - */ - abstract getSliceStates(): Promise - - /** - * @param hashes Slices that are no longer part of config - */ - abstract removeSlices(hashes: SliceHash[]): Promise - - /** - * - * @param slices Slices that are part of config and need to be updated - * @returns Minimum of the heights of updated slices - */ - abstract updateSlices(slices: SliceUpdate[]): Promise -} - -/** - * slices toRemove are slices that are no longer part of config - */ -export function diffSlices( - expectedSlices: SliceHash[], - actualSlices: SliceState[], - from: number, - to: number, -): { toUpdate: SliceUpdate[]; toRemove: SliceHash[] } { - const toUpdate: SliceUpdate[] = [] - const toRemove: SliceHash[] = [] - - for (const slice of actualSlices) { - if (!expectedSlices.includes(slice.sliceHash)) { - toRemove.push(slice.sliceHash) - } else if (slice.height < to) { - toUpdate.push({ - sliceHash: slice.sliceHash, - from: Math.max(slice.height, from), - to: to, - }) - } - } - - for (const slice of expectedSlices) { - if (!actualSlices.find((s) => s.sliceHash === slice)) { - toUpdate.push({ sliceHash: slice, from: from, to: to }) - } - } - - return { toUpdate, toRemove } -} diff --git a/packages/uif/src/index.ts b/packages/uif/src/index.ts index b4968e6d..9d6cc6d1 100644 --- a/packages/uif/src/index.ts +++ b/packages/uif/src/index.ts @@ -1,4 +1,6 @@ -export * from './BaseIndexer' export * from './Indexer' +export * from './indexers/ChildIndexer' +export * from './indexers/multi/MultiIndexer' +export * from './indexers/multi/types' +export * from './indexers/RootIndexer' export * from './Retries' -export * from './SliceIndexer' diff --git a/packages/uif/src/indexers/ChildIndexer.ts b/packages/uif/src/indexers/ChildIndexer.ts new file mode 100644 index 00000000..7d4e2d53 --- /dev/null +++ b/packages/uif/src/indexers/ChildIndexer.ts @@ -0,0 +1,7 @@ +import { Indexer } from '../Indexer' + +export abstract class ChildIndexer extends Indexer { + override async tick(): Promise { + return Promise.reject(new Error('ChildIndexer cannot tick')) + } +} diff --git a/packages/uif/src/indexers/RootIndexer.ts b/packages/uif/src/indexers/RootIndexer.ts new file mode 100644 index 00000000..923964a8 --- /dev/null +++ b/packages/uif/src/indexers/RootIndexer.ts @@ -0,0 +1,21 @@ +import { Logger } from '@l2beat/backend-tools' + +import { Indexer, IndexerOptions } from '../Indexer' + +export abstract class RootIndexer extends Indexer { + constructor(logger: Logger, opts?: IndexerOptions) { + super(logger, [], opts) + } + + override async update(): Promise { + return Promise.reject(new Error('RootIndexer cannot update')) + } + + override async invalidate(): Promise { + return Promise.reject(new Error('RootIndexer cannot invalidate')) + } + + override async setSafeHeight(): Promise { + return Promise.resolve() + } +} diff --git a/packages/uif/src/indexers/multi/MultiIndexer.test.ts b/packages/uif/src/indexers/multi/MultiIndexer.test.ts new file mode 100644 index 00000000..d1820124 --- /dev/null +++ b/packages/uif/src/indexers/multi/MultiIndexer.test.ts @@ -0,0 +1,443 @@ +import { Logger } from '@l2beat/backend-tools' +import { expect, mockFn } from 'earl' + +import { MultiIndexer } from './MultiIndexer' +import { + Configuration, + RemovalConfiguration, + SavedConfiguration, + UpdateConfiguration, +} from './types' + +describe(MultiIndexer.name, () => { + describe(MultiIndexer.prototype.initialize.name, () => { + it('calls multiInitialize and saves configurations', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 400), actual('b', 200, 500)], + [ + saved('a', 100, 400, 300), + saved('b', 200, 500, 300), + saved('c', 100, 300, 300), + ], + ) + + const newHeight = await testIndexer.initialize() + expect(newHeight).toEqual(300) + + expect(testIndexer.removeData).toHaveBeenOnlyCalledWith([ + removal('c', 100, 300), + ]) + expect(testIndexer.saveConfigurations).toHaveBeenOnlyCalledWith([ + saved('a', 100, 400, 300), + saved('b', 200, 500, 300), + ]) + }) + + it('skips calling removeData if there is nothing to remove', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 400), actual('b', 200, 500)], + [saved('a', 100, 400, 400), saved('b', 200, 500, 500)], + ) + + const newHeight = await testIndexer.initialize() + expect(newHeight).toEqual(Infinity) + + expect(testIndexer.removeData).not.toHaveBeenCalled() + expect(testIndexer.saveConfigurations).toHaveBeenCalledWith([ + saved('a', 100, 400, 400), + saved('b', 200, 500, 500), + ]) + }) + + it('no synced data', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 400), actual('b', 200, null)], + [], + ) + + const newHeight = await testIndexer.initialize() + expect(newHeight).toEqual(99) + + expect(testIndexer.removeData).not.toHaveBeenCalled() + expect(testIndexer.saveConfigurations).toHaveBeenCalledWith([ + saved('a', 100, 400, null), + saved('b', 200, null, null), + ]) + }) + + it('mismatched min and max times', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 500), actual('b', 200, 400), actual('c', 300, null)], + [ + saved('a', 100, 400, 300), + saved('b', 100, 300, 300), + saved('c', 300, 400, 300), + ], + ) + + const newHeight = await testIndexer.initialize() + expect(newHeight).toEqual(300) + + expect(testIndexer.removeData).toHaveBeenCalledWith([ + removal('b', 100, 199), + ]) + expect(testIndexer.saveConfigurations).toHaveBeenCalledWith([ + saved('a', 100, 500, 300), + saved('b', 200, 400, 300), + saved('c', 300, null, 300), + ]) + }) + }) + + describe(MultiIndexer.prototype.update.name, () => { + it('calls multiUpdate with an early matching configuration', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 200), actual('b', 300, 400)], + [], + ) + await testIndexer.initialize() + + const newHeight = await testIndexer.update(100, 500) + + expect(newHeight).toEqual(200) + expect(testIndexer.multiUpdate).toHaveBeenOnlyCalledWith(100, 200, [ + update('a', 100, 200, false), + ]) + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(2, [ + saved('a', 100, 200, 200), + saved('b', 300, 400, null), + ]) + }) + + it('calls multiUpdate with a late matching configuration', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 200), actual('b', 300, 400)], + [saved('a', 100, 200, 200)], + ) + await testIndexer.initialize() + + const newHeight = await testIndexer.update(300, 500) + + expect(newHeight).toEqual(400) + expect(testIndexer.multiUpdate).toHaveBeenOnlyCalledWith(300, 400, [ + update('b', 300, 400, false), + ]) + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(2, [ + saved('a', 100, 200, 200), + saved('b', 300, 400, 400), + ]) + }) + + it('calls multiUpdate with two matching configurations', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 200), actual('b', 100, 400)], + [], + ) + await testIndexer.initialize() + + const newHeight = await testIndexer.update(100, 500) + + expect(newHeight).toEqual(200) + expect(testIndexer.multiUpdate).toHaveBeenOnlyCalledWith(100, 200, [ + update('a', 100, 200, false), + update('b', 100, 400, false), + ]) + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(2, [ + saved('a', 100, 200, 200), + saved('b', 100, 400, 200), + ]) + }) + + it('calls multiUpdate with two middle matching configurations', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 400), actual('b', 200, 500)], + [saved('a', 100, 400, 300), saved('b', 200, 500, 300)], + ) + await testIndexer.initialize() + + const newHeight = await testIndexer.update(301, 600) + + expect(newHeight).toEqual(400) + expect(testIndexer.multiUpdate).toHaveBeenOnlyCalledWith(301, 400, [ + update('a', 100, 400, false), + update('b', 200, 500, false), + ]) + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(2, [ + saved('a', 100, 400, 400), + saved('b', 200, 500, 400), + ]) + }) + + it('skips calling multiUpdate if we are too early', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 200), actual('b', 300, 400)], + [], + ) + await testIndexer.initialize() + + const newHeight = await testIndexer.update(0, 500) + + expect(newHeight).toEqual(99) + expect(testIndexer.multiUpdate).not.toHaveBeenCalled() + expect(testIndexer.saveConfigurations).toHaveBeenCalledTimes(1) + }) + + it('skips calling multiUpdate if we are too late', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 200), actual('b', 300, 400)], + [], + ) + await testIndexer.initialize() + + const newHeight = await testIndexer.update(401, 500) + + expect(newHeight).toEqual(500) + expect(testIndexer.multiUpdate).not.toHaveBeenCalled() + expect(testIndexer.saveConfigurations).toHaveBeenCalledTimes(1) + }) + + it('skips calling multiUpdate between configs', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 200), actual('b', 300, 400)], + [], + ) + await testIndexer.initialize() + + const newHeight = await testIndexer.update(201, 500) + + expect(newHeight).toEqual(299) + expect(testIndexer.multiUpdate).not.toHaveBeenCalled() + expect(testIndexer.saveConfigurations).toHaveBeenCalledTimes(1) + }) + + it('calls multiUpdate with a matching configuration with data', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 200), actual('b', 100, 400)], + [saved('a', 100, 200, 200)], + ) + await testIndexer.initialize() + + const newHeight = await testIndexer.update(100, 500) + + expect(newHeight).toEqual(200) + expect(testIndexer.multiUpdate).toHaveBeenOnlyCalledWith(100, 200, [ + update('a', 100, 200, true), + update('b', 100, 400, false), + ]) + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(2, [ + saved('a', 100, 200, 200), + saved('b', 100, 400, 200), + ]) + }) + + it('multiple update calls', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 200), actual('b', 100, 400)], + [saved('a', 100, 200, 200)], + ) + await testIndexer.initialize() + + expect(await testIndexer.update(100, 500)).toEqual(200) + expect(testIndexer.multiUpdate).toHaveBeenNthCalledWith(1, 100, 200, [ + update('a', 100, 200, true), + update('b', 100, 400, false), + ]) + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(2, [ + saved('a', 100, 200, 200), + saved('b', 100, 400, 200), + ]) + + // The same range. In real life might be a result of a parent reorg + // Invalidate is a no-op so we don't need to call it + expect(await testIndexer.update(100, 500)).toEqual(200) + expect(testIndexer.multiUpdate).toHaveBeenNthCalledWith(2, 100, 200, [ + update('a', 100, 200, true), + update('b', 100, 400, true), + ]) + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(3, [ + saved('a', 100, 200, 200), + saved('b', 100, 400, 200), + ]) + + // Next range + expect(await testIndexer.update(201, 500)).toEqual(400) + expect(testIndexer.multiUpdate).toHaveBeenNthCalledWith(3, 201, 400, [ + update('b', 100, 400, false), + ]) + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(4, [ + saved('a', 100, 200, 200), + saved('b', 100, 400, 400), + ]) + }) + + it('correctly updates currentHeight in saved configurations', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 500), actual('b', 100, 500), actual('c', 100, 500)], + [ + saved('a', 100, 500, null), + saved('b', 100, 500, 250), + saved('c', 100, 500, 500), + ], + ) + expect(await testIndexer.initialize()).toEqual(99) + + expect(await testIndexer.update(100, 500)).toEqual(250) + expect(testIndexer.multiUpdate).toHaveBeenNthCalledWith(1, 100, 250, [ + update('a', 100, 500, false), + update('b', 100, 500, true), + update('c', 100, 500, true), + ]) + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(2, [ + saved('a', 100, 500, 250), + saved('b', 100, 500, 250), + saved('c', 100, 500, 500), + ]) + + expect(await testIndexer.update(251, 500)).toEqual(500) + expect(testIndexer.multiUpdate).toHaveBeenNthCalledWith(2, 251, 500, [ + update('a', 100, 500, false), + update('b', 100, 500, false), + update('c', 100, 500, true), + ]) + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(3, [ + saved('a', 100, 500, 500), + saved('b', 100, 500, 500), + saved('c', 100, 500, 500), + ]) + }) + }) + + describe('multiUpdate', () => { + it('returns the currentHeight', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 300), actual('b', 100, 400)], + [saved('a', 100, 300, 200), saved('b', 100, 400, 200)], + ) + await testIndexer.initialize() + + testIndexer.multiUpdate.resolvesTo(200) + + const newHeight = await testIndexer.update(200, 500) + expect(newHeight).toEqual(200) + expect(testIndexer.saveConfigurations).toHaveBeenCalledTimes(1) + }) + + it('returns the targetHeight', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 300), actual('b', 100, 400)], + [saved('a', 100, 300, 200), saved('b', 100, 400, 200)], + ) + await testIndexer.initialize() + + testIndexer.multiUpdate.resolvesTo(300) + + const newHeight = await testIndexer.update(200, 300) + expect(newHeight).toEqual(300) + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(2, [ + saved('a', 100, 300, 300), + saved('b', 100, 400, 300), + ]) + }) + + it('returns something in between', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 300), actual('b', 100, 400)], + [saved('a', 100, 300, 200), saved('b', 100, 400, 200)], + ) + await testIndexer.initialize() + + testIndexer.multiUpdate.resolvesTo(250) + + const newHeight = await testIndexer.update(200, 300) + expect(newHeight).toEqual(250) + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(2, [ + saved('a', 100, 300, 250), + saved('b', 100, 400, 250), + ]) + }) + + it('cannot return less than currentHeight', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 300), actual('b', 100, 400)], + [saved('a', 100, 300, 200), saved('b', 100, 400, 200)], + ) + await testIndexer.initialize() + + testIndexer.multiUpdate.resolvesTo(150) + + await expect(testIndexer.update(200, 300)).toBeRejectedWith( + /returned height must be between from and to/, + ) + }) + + it('cannot return more than targetHeight', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 300), actual('b', 100, 400)], + [saved('a', 100, 300, 200), saved('b', 100, 400, 200)], + ) + await testIndexer.initialize() + + testIndexer.multiUpdate.resolvesTo(350) + + await expect(testIndexer.update(200, 300)).toBeRejectedWith( + /returned height must be between from and to/, + ) + }) + }) +}) + +class TestMultiIndexer extends MultiIndexer { + constructor( + configurations: Configuration[], + private readonly _saved: SavedConfiguration[], + ) { + super(Logger.SILENT, [], configurations) + } + + override multiInitialize(): Promise[]> { + return Promise.resolve(this._saved) + } + + multiUpdate = mockFn['multiUpdate']>((_, targetHeight) => + Promise.resolve(targetHeight), + ) + + removeData = mockFn['removeData']>().resolvesTo(undefined) + + saveConfigurations = + mockFn['saveConfigurations']>().resolvesTo(undefined) +} + +function actual( + id: string, + minHeight: number, + maxHeight: number | null, +): Configuration { + return { id, properties: null, minHeight, maxHeight } +} + +function saved( + id: string, + minHeight: number, + maxHeight: number | null, + currentHeight: number | null, +): SavedConfiguration { + return { id, properties: null, minHeight, maxHeight, currentHeight } +} + +function update( + id: string, + minHeight: number, + maxHeight: number | null, + hasData: boolean, +): UpdateConfiguration { + return { id, properties: null, minHeight, maxHeight, hasData } +} + +function removal( + id: string, + from: number, + to: number, +): RemovalConfiguration { + return { id, properties: null, from, to } +} diff --git a/packages/uif/src/indexers/multi/MultiIndexer.ts b/packages/uif/src/indexers/multi/MultiIndexer.ts new file mode 100644 index 00000000..10935a0e --- /dev/null +++ b/packages/uif/src/indexers/multi/MultiIndexer.ts @@ -0,0 +1,202 @@ +import { Logger } from '@l2beat/backend-tools' + +import { Indexer, IndexerOptions } from '../../Indexer' +import { ChildIndexer } from '../ChildIndexer' +import { diffConfigurations } from './diffConfigurations' +import { toRanges } from './toRanges' +import { + Configuration, + ConfigurationRange, + RemovalConfiguration, + SavedConfiguration, + UpdateConfiguration, +} from './types' + +export abstract class MultiIndexer extends ChildIndexer { + private readonly ranges: ConfigurationRange[] + private saved: SavedConfiguration[] = [] + + constructor( + logger: Logger, + parents: Indexer[], + readonly configurations: Configuration[], + options?: IndexerOptions, + ) { + super(logger, parents, options) + this.ranges = toRanges(configurations) + } + + /** + * Initializes the indexer. It returns the configurations that were saved + * previously. In case no configurations were saved, it should return an empty + * array. + * + * This method is expected to read the configurations that was saved + * previously with `setStoredConfigurations`. It shouldn't call + * `setStoredConfigurations` itself. + * + * @returns The configurations that were saved previously. + */ + abstract multiInitialize(): Promise[]> + + /** + * Implements the main data fetching process. It is up to the indexer to + * decide how much data to fetch. For example given `.update(100, 200, [...])`, the + * indexer can only fetch data up to 110 and return 110. The next time this + * method will be called with `.update(110, 200, [...])`. + * + * @param from The height for which the indexer should start syncing data. + * This value is inclusive. If the indexer hasn't synced anything previously + * this will equal the minimum height of all configurations. + * + * @param to The height at which the indexer should end syncing data. This + * value is also inclusive so the indexer should eventually sync data for this + * height. + * + * @param configurations The configurations that the indexer should use to + * sync data. The configurations are guaranteed to be in the range of + * `from` and `to`. Some of those configurations might have been synced + * previously for this range. Those configurations will include the `hasData` + * flag set to `true`. + * + * @returns The height that the indexer has synced up to. Returning + * `from` means that the indexer has synced a single data point. Returning + * a value greater than `from` means that the indexer has synced up + * to that height. Returning a value less than `from` or greater than + * `to` is not permitted. + */ + abstract multiUpdate( + from: number, + to: number, + configurations: UpdateConfiguration[], + ): Promise + + /** + * Removes data that was previously synced but because configurations changed + * is no longer valid. The data should be removed for the ranges specified + * in each configuration. It is possible for multiple ranges to share a + * configuration id! + * + * This method can only be called during the initialization of the indexer, + * after `multiInitialize` returns. + */ + abstract removeData(configurations: RemovalConfiguration[]): Promise + + /** + * Saves configurations that the indexer should use to sync data. The + * configurations saved here should be read in the `multiInitialize` method. + * + * @param configurations The configurations that the indexer should save. The + * indexer should save the returned configurations and ensure that no other + * configurations are persisted. + */ + abstract saveConfigurations( + configurations: SavedConfiguration[], + ): Promise + + async initialize(): Promise { + const saved = await this.multiInitialize() + const { toRemove, toSave, safeHeight } = diffConfigurations( + this.configurations, + saved, + ) + this.saved = toSave + if (toRemove.length > 0) { + await this.removeData(toRemove) + } + await this.saveConfigurations(toSave) + return safeHeight + } + + async update(from: number, to: number): Promise { + const range = findRange(this.ranges, from) + if (range.configurations.length === 0) { + return Math.min(range.to, to) + } + + const { configurations, minCurrentHeight } = getConfigurationsInRange( + range, + this.saved, + from, + ) + const adjustedTo = Math.min(range.to, to, minCurrentHeight) + + this.logger.info('Calling multiUpdate', { + from, + to: adjustedTo, + configurations: configurations.length, + }) + const newHeight = await this.multiUpdate(from, adjustedTo, configurations) + if (newHeight < from || newHeight > adjustedTo) { + throw new Error( + 'Programmer error, returned height must be between from and to (both inclusive).', + ) + } + + if (newHeight > from) { + this.updateSavedConfigurations(configurations, newHeight) + await this.saveConfigurations(this.saved) + } + + return newHeight + } + + private updateSavedConfigurations( + updatedConfigurations: UpdateConfiguration[], + newHeight: number, + ): void { + for (const updated of updatedConfigurations) { + const saved = this.saved.find((c) => c.id === updated.id) + if (!saved) { + throw new Error('Programmer error, saved configuration not found') + } + if (saved.currentHeight === null || saved.currentHeight < newHeight) { + saved.currentHeight = newHeight + } + } + } + + async invalidate(targetHeight: number): Promise { + return Promise.resolve(targetHeight) + } + + async setSafeHeight(): Promise { + return Promise.resolve() + } +} + +function findRange( + ranges: ConfigurationRange[], + from: number, +): ConfigurationRange { + const range = ranges.find((range) => range.from <= from && range.to >= from) + if (!range) { + throw new Error('Programmer error, there should always be a range') + } + return range +} + +function getConfigurationsInRange( + range: ConfigurationRange, + savedConfigurations: SavedConfiguration[], + currentHeight: number, +): { configurations: UpdateConfiguration[]; minCurrentHeight: number } { + let minCurrentHeight = Infinity + const configurations = range.configurations.map( + (configuration): UpdateConfiguration => { + const saved = savedConfigurations.find((c) => c.id === configuration.id) + if ( + // eslint-disable-next-line @typescript-eslint/prefer-optional-chain + saved && + saved.currentHeight !== null && + saved.currentHeight > currentHeight + ) { + minCurrentHeight = Math.min(minCurrentHeight, saved.currentHeight) + return { ...configuration, hasData: true } + } else { + return { ...configuration, hasData: false } + } + }, + ) + return { configurations, minCurrentHeight } +} diff --git a/packages/uif/src/indexers/multi/diffConfigurations.test.ts b/packages/uif/src/indexers/multi/diffConfigurations.test.ts new file mode 100644 index 00000000..bb745745 --- /dev/null +++ b/packages/uif/src/indexers/multi/diffConfigurations.test.ts @@ -0,0 +1,226 @@ +import { expect } from 'earl' + +import { diffConfigurations } from './diffConfigurations' +import { + Configuration, + RemovalConfiguration, + SavedConfiguration, +} from './types' + +describe(diffConfigurations.name, () => { + describe('errors', () => { + it('duplicate config id', () => { + expect(() => + diffConfigurations([actual('a', 100, null), actual('a', 200, 300)], []), + ).toThrow(/a is duplicated/) + }) + + it('minHeight greater than maxHeight', () => { + expect(() => diffConfigurations([actual('a', 200, 100)], [])).toThrow( + /a has minHeight greater than maxHeight/, + ) + }) + }) + + describe('regular sync', () => { + it('empty actual and stored', () => { + const result = diffConfigurations([], []) + expect(result).toEqual({ toRemove: [], toSave: [], safeHeight: Infinity }) + }) + + it('empty stored', () => { + const result = diffConfigurations( + [actual('a', 100, null), actual('b', 200, 300)], + [], + ) + expect(result).toEqual({ + toRemove: [], + toSave: [saved('a', 100, null, null), saved('b', 200, 300, null)], + safeHeight: 99, + }) + }) + + it('partially synced, both early', () => { + const result = diffConfigurations( + [actual('a', 100, 400), actual('b', 200, null)], + [saved('a', 100, 400, 300), saved('b', 200, null, 300)], + ) + expect(result).toEqual({ + toRemove: [], + toSave: [saved('a', 100, 400, 300), saved('b', 200, null, 300)], + safeHeight: 300, + }) + }) + + it('partially synced, one new not yet started', () => { + const result = diffConfigurations( + [actual('a', 100, 400), actual('b', 555, null)], + [saved('a', 100, 400, 300), saved('b', 555, null, null)], + ) + expect(result).toEqual({ + toRemove: [], + toSave: [saved('a', 100, 400, 300), saved('b', 555, null, null)], + safeHeight: 300, + }) + }) + + it('partially synced, one finished', () => { + const result = diffConfigurations( + [actual('a', 100, 555), actual('b', 200, 300)], + [saved('a', 100, 555, 400), saved('b', 200, 300, 300)], + ) + expect(result).toEqual({ + toRemove: [], + toSave: [saved('a', 100, 555, 400), saved('b', 200, 300, 300)], + safeHeight: 400, + }) + }) + + it('partially synced, one finished, one infinite', () => { + const result = diffConfigurations( + [actual('a', 100, null), actual('b', 200, 300)], + [saved('a', 100, null, 400), saved('b', 200, 300, 300)], + ) + expect(result).toEqual({ + toRemove: [], + toSave: [saved('a', 100, null, 400), saved('b', 200, 300, 300)], + safeHeight: 400, + }) + }) + + it('both synced', () => { + const result = diffConfigurations( + [actual('a', 100, 400), actual('b', 200, 300)], + [saved('a', 100, 400, 400), saved('b', 200, 300, 300)], + ) + expect(result).toEqual({ + toRemove: [], + toSave: [saved('a', 100, 400, 400), saved('b', 200, 300, 300)], + safeHeight: Infinity, + }) + }) + }) + + describe('configuration changed', () => { + it('empty actual', () => { + const result = diffConfigurations( + [], + [saved('a', 100, 400, 300), saved('b', 200, null, 300)], + ) + expect(result).toEqual({ + toRemove: [removal('a', 100, 300), removal('b', 200, 300)], + toSave: [], + safeHeight: Infinity, + }) + }) + + it('single removed', () => { + const result = diffConfigurations( + [actual('b', 200, 400)], + [saved('a', 100, null, 300), saved('b', 200, 400, 300)], + ) + expect(result).toEqual({ + toRemove: [removal('a', 100, 300)], + toSave: [saved('b', 200, 400, 300)], + safeHeight: 300, + }) + }) + + it('maxHeight updated up', () => { + const result = diffConfigurations( + [actual('a', 100, 400)], + [saved('a', 100, 300, 300)], + ) + expect(result).toEqual({ + toRemove: [], + toSave: [saved('a', 100, 400, 300)], + safeHeight: 300, + }) + }) + + it('maxHeight updated down', () => { + const result = diffConfigurations( + [actual('a', 100, 200)], + [saved('a', 100, 300, 300)], + ) + expect(result).toEqual({ + toRemove: [removal('a', 201, 300)], + toSave: [saved('a', 100, 200, 200)], + safeHeight: Infinity, + }) + }) + + it('maxHeight removed', () => { + const result = diffConfigurations( + [actual('a', 100, null)], + [saved('a', 100, 300, 300)], + ) + expect(result).toEqual({ + toRemove: [], + toSave: [saved('a', 100, null, 300)], + safeHeight: 300, + }) + }) + + it('minHeight updated up', () => { + const result = diffConfigurations( + [actual('a', 200, 400)], + [saved('a', 100, 400, 300)], + ) + expect(result).toEqual({ + toRemove: [removal('a', 100, 199)], + toSave: [saved('a', 200, 400, 300)], + safeHeight: 300, + }) + }) + + it('minHeight updated down', () => { + const result = diffConfigurations( + [actual('a', 100, 400)], + [saved('a', 200, 400, 300)], + ) + expect(result).toEqual({ + toRemove: [removal('a', 200, 300)], + toSave: [saved('a', 100, 400, null)], + safeHeight: 99, + }) + }) + + it('both min and max height updated', () => { + const result = diffConfigurations( + [actual('a', 200, 300)], + [saved('a', 100, 400, 400)], + ) + expect(result).toEqual({ + toRemove: [removal('a', 100, 199), removal('a', 301, 400)], + toSave: [saved('a', 200, 300, 300)], + safeHeight: Infinity, + }) + }) + }) +}) + +function actual( + id: string, + minHeight: number, + maxHeight: number | null, +): Configuration { + return { id, properties: null, minHeight, maxHeight } +} + +function saved( + id: string, + minHeight: number, + maxHeight: number | null, + currentHeight: number | null, +): SavedConfiguration { + return { id, properties: null, minHeight, maxHeight, currentHeight } +} + +function removal( + id: string, + from: number, + to: number, +): RemovalConfiguration { + return { id, properties: null, from, to } +} diff --git a/packages/uif/src/indexers/multi/diffConfigurations.ts b/packages/uif/src/indexers/multi/diffConfigurations.ts new file mode 100644 index 00000000..92bba679 --- /dev/null +++ b/packages/uif/src/indexers/multi/diffConfigurations.ts @@ -0,0 +1,97 @@ +import { + Configuration, + RemovalConfiguration, + SavedConfiguration, +} from './types' + +export function diffConfigurations( + actual: Configuration[], + saved: SavedConfiguration[], +): { + toRemove: RemovalConfiguration[] + toSave: SavedConfiguration[] + safeHeight: number +} { + let safeHeight = Infinity + + const actualMap = new Map(actual.map((c) => [c.id, c])) + const savedMap = new Map(saved.map((c) => [c.id, c])) + + const toRemove: RemovalConfiguration[] = [] + for (const c of saved) { + if (actualMap.has(c.id) || c.currentHeight === null) { + continue + } + toRemove.push({ + id: c.id, + properties: c.properties, + from: c.minHeight, + to: c.currentHeight, + }) + } + + const toSave: SavedConfiguration[] = [] + + const knownIds = new Set() + for (const c of actual) { + if (knownIds.has(c.id)) { + throw new Error(`Configuration ${c.id} is duplicated!`) + } + knownIds.add(c.id) + + if (c.maxHeight !== null && c.minHeight > c.maxHeight) { + throw new Error( + `Configuration ${c.id} has minHeight greater than maxHeight!`, + ) + } + + const stored = savedMap.get(c.id) + if (!stored || stored.currentHeight === null) { + safeHeight = Math.min(safeHeight, c.minHeight - 1) + toSave.push({ ...c, currentHeight: null }) + continue + } + + if (stored.minHeight > c.minHeight) { + safeHeight = Math.min(safeHeight, c.minHeight - 1) + // We remove everything because we cannot have gaps in downloaded data + // We will re-download everything from the beginning + toRemove.push({ + id: stored.id, + properties: stored.properties, + from: stored.minHeight, + to: stored.currentHeight, + }) + toSave.push({ ...c, currentHeight: null }) + continue + } + + if (stored.minHeight < c.minHeight) { + toRemove.push({ + id: stored.id, + properties: stored.properties, + from: stored.minHeight, + to: c.minHeight - 1, + }) + } + + if (c.maxHeight !== null && stored.currentHeight > c.maxHeight) { + toRemove.push({ + id: stored.id, + properties: stored.properties, + from: c.maxHeight + 1, + to: stored.currentHeight, + }) + } else if (c.maxHeight === null || stored.currentHeight < c.maxHeight) { + safeHeight = Math.min(safeHeight, stored.currentHeight) + } + + const currentHeight = Math.min( + stored.currentHeight, + c.maxHeight ?? stored.currentHeight, + ) + toSave.push({ ...c, currentHeight }) + } + + return { toRemove, toSave, safeHeight } +} diff --git a/packages/uif/src/indexers/multi/toRanges.test.ts b/packages/uif/src/indexers/multi/toRanges.test.ts new file mode 100644 index 00000000..f27b189f --- /dev/null +++ b/packages/uif/src/indexers/multi/toRanges.test.ts @@ -0,0 +1,232 @@ +import { expect } from 'earl' + +import { toRanges } from './toRanges' +import { Configuration } from './types' + +describe(toRanges.name, () => { + it('empty', () => { + const ranges = toRanges([]) + expect(ranges).toEqual([ + { from: -Infinity, to: Infinity, configurations: [] }, + ]) + }) + + it('single infinite configuration', () => { + const ranges = toRanges([actual('a', 100, null)]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { from: 100, to: Infinity, configurations: [actual('a', 100, null)] }, + ]) + }) + + it('single finite configuration', () => { + const ranges = toRanges([actual('a', 100, 300)]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { from: 100, to: 300, configurations: [actual('a', 100, 300)] }, + { from: 301, to: Infinity, configurations: [] }, + ]) + }) + + it('multiple overlapping configurations on the edges', () => { + const ranges = toRanges([actual('a', 100, 300), actual('b', 300, 500)]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { from: 100, to: 299, configurations: [actual('a', 100, 300)] }, + { + from: 300, + to: 300, + configurations: [actual('a', 100, 300), actual('b', 300, 500)], + }, + { from: 301, to: 500, configurations: [actual('b', 300, 500)] }, + { from: 501, to: Infinity, configurations: [] }, + ]) + }) + + it('multiple overlapping configurations', () => { + const ranges = toRanges([ + actual('a', 100, 300), + actual('b', 200, 400), + actual('c', 300, 500), + ]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { from: 100, to: 199, configurations: [actual('a', 100, 300)] }, + { + from: 200, + to: 299, + configurations: [actual('a', 100, 300), actual('b', 200, 400)], + }, + { + from: 300, + to: 300, + configurations: [ + actual('a', 100, 300), + actual('b', 200, 400), + actual('c', 300, 500), + ], + }, + { + from: 301, + to: 400, + configurations: [actual('b', 200, 400), actual('c', 300, 500)], + }, + { from: 401, to: 500, configurations: [actual('c', 300, 500)] }, + { from: 501, to: Infinity, configurations: [] }, + ]) + }) + + it('multiple non-overlapping configurations', () => { + const ranges = toRanges([ + actual('a', 100, 200), + actual('b', 300, 400), + actual('c', 500, 600), + ]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { from: 100, to: 200, configurations: [actual('a', 100, 200)] }, + { from: 201, to: 299, configurations: [] }, + { from: 300, to: 400, configurations: [actual('b', 300, 400)] }, + { from: 401, to: 499, configurations: [] }, + { from: 500, to: 600, configurations: [actual('c', 500, 600)] }, + { from: 601, to: Infinity, configurations: [] }, + ]) + }) + + it('multiple overlapping and non-overlapping configurations', () => { + const ranges = toRanges([ + actual('a', 100, 200), + actual('b', 300, 500), + actual('c', 400, 600), + actual('d', 700, 800), + ]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { from: 100, to: 200, configurations: [actual('a', 100, 200)] }, + { from: 201, to: 299, configurations: [] }, + { from: 300, to: 399, configurations: [actual('b', 300, 500)] }, + { + from: 400, + to: 500, + configurations: [actual('b', 300, 500), actual('c', 400, 600)], + }, + { from: 501, to: 600, configurations: [actual('c', 400, 600)] }, + { from: 601, to: 699, configurations: [] }, + { from: 700, to: 800, configurations: [actual('d', 700, 800)] }, + { from: 801, to: Infinity, configurations: [] }, + ]) + }) + + it('adjacent: one configuration start where other ends', () => { + const ranges = toRanges([actual('a', 100, 200), actual('b', 200, 300)]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { from: 100, to: 199, configurations: [actual('a', 100, 200)] }, + { + from: 200, + to: 200, + configurations: [actual('a', 100, 200), actual('b', 200, 300)], + }, + { + from: 201, + to: 300, + configurations: [actual('b', 200, 300)], + }, + { + from: 301, + to: Infinity, + configurations: [], + }, + ]) + }) + + it('identical: two configurations with exactly the same boundaries', () => { + const ranges = toRanges([actual('a', 100, 200), actual('b', 100, 200)]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { + from: 100, + to: 200, + configurations: [actual('a', 100, 200), actual('b', 100, 200)], + }, + { + from: 201, + to: Infinity, + configurations: [], + }, + ]) + }) + + it('single point: configuration starts and ends in the same time', () => { + const ranges = toRanges([actual('a', 100, 100)]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { + from: 100, + to: 100, + configurations: [actual('a', 100, 100)], + }, + { + from: 101, + to: Infinity, + configurations: [], + }, + ]) + }) + + it('order of inputs does not affect output', () => { + const ranges = toRanges([ + actual('b', 300, 400), + actual('c', 500, 600), + actual('a', 100, 200), + ]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { from: 100, to: 200, configurations: [actual('a', 100, 200)] }, + { from: 201, to: 299, configurations: [] }, + { from: 300, to: 400, configurations: [actual('b', 300, 400)] }, + { from: 401, to: 499, configurations: [] }, + { from: 500, to: 600, configurations: [actual('c', 500, 600)] }, + { from: 601, to: Infinity, configurations: [] }, + ]) + }) + + it('same starting point, multiple maxHeights', () => { + const ranges = toRanges([ + actual('a', 100, 200), + actual('b', 100, 300), + actual('c', 100, 400), + ]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { + from: 100, + to: 200, + configurations: [ + actual('a', 100, 200), + actual('b', 100, 300), + actual('c', 100, 400), + ], + }, + { + from: 201, + to: 300, + configurations: [actual('b', 100, 300), actual('c', 100, 400)], + }, + { + from: 301, + to: 400, + configurations: [actual('c', 100, 400)], + }, + { from: 401, to: Infinity, configurations: [] }, + ]) + }) +}) + +function actual( + id: string, + minHeight: number, + maxHeight: number | null, +): Configuration { + return { id, properties: null, minHeight, maxHeight } +} diff --git a/packages/uif/src/indexers/multi/toRanges.ts b/packages/uif/src/indexers/multi/toRanges.ts new file mode 100644 index 00000000..a99e94b2 --- /dev/null +++ b/packages/uif/src/indexers/multi/toRanges.ts @@ -0,0 +1,44 @@ +import { Configuration, ConfigurationRange } from './types' + +export function toRanges( + configurations: Configuration[], +): ConfigurationRange[] { + const minHeights = configurations.map((c) => c.minHeight) + const maxHeights = configurations + .map((c) => c.maxHeight) + .filter((height): height is number => height !== null) + + const starts = minHeights + .concat(maxHeights.map((height) => height + 1)) + .sort((a, b) => a - b) + .filter((height, i, arr) => arr.indexOf(height) === i) + + let lastRange: ConfigurationRange = { + from: -Infinity, + to: Infinity, + configurations: [], + } + const ranges: ConfigurationRange[] = [lastRange] + for (const start of starts) { + lastRange.to = start - 1 + lastRange = { + from: start, + to: Infinity, + configurations: [], + } + ranges.push(lastRange) + } + + for (const configuration of configurations) { + const min = configuration.minHeight + const max = configuration.maxHeight ?? Infinity + + for (const range of ranges) { + if (!(max < range.from || min > range.to)) { + range.configurations.push(configuration) + } + } + } + + return ranges +} diff --git a/packages/uif/src/indexers/multi/types.ts b/packages/uif/src/indexers/multi/types.ts new file mode 100644 index 00000000..b2079f5f --- /dev/null +++ b/packages/uif/src/indexers/multi/types.ts @@ -0,0 +1,33 @@ +export interface Configuration { + id: string + properties: T + /** Inclusive */ + minHeight: number + /** Inclusive */ + maxHeight: number | null +} + +export interface UpdateConfiguration extends Configuration { + hasData: boolean +} + +export interface SavedConfiguration extends Configuration { + currentHeight: number | null +} + +export interface RemovalConfiguration { + id: string + properties: T + /** Inclusive */ + from: number + /** Inclusive */ + to: number +} + +export interface ConfigurationRange { + /** Inclusive */ + from: number + /** Inclusive */ + to: number + configurations: Configuration[] +} diff --git a/packages/uif/src/reducer/handlers/handleTickSucceeded.ts b/packages/uif/src/reducer/handlers/handleTickSucceeded.ts index c3b41e91..c1fc15d0 100644 --- a/packages/uif/src/reducer/handlers/handleTickSucceeded.ts +++ b/packages/uif/src/reducer/handlers/handleTickSucceeded.ts @@ -11,9 +11,10 @@ export function handleTickSucceeded( ): IndexerReducerResult { assertRoot(state) assertStatus(state.status, 'ticking') - const effects: IndexerEffect[] = [ - { type: 'SetSafeHeight', safeHeight: action.safeHeight }, - ] + const effects: IndexerEffect[] = + action.safeHeight !== state.safeHeight + ? [{ type: 'SetSafeHeight', safeHeight: action.safeHeight }] + : [] if (state.tickScheduled) { effects.push({ type: 'Tick' }) } diff --git a/packages/uif/src/reducer/handlers/handleUpdateSucceeded.ts b/packages/uif/src/reducer/handlers/handleUpdateSucceeded.ts index 39ea011a..dcd451d3 100644 --- a/packages/uif/src/reducer/handlers/handleUpdateSucceeded.ts +++ b/packages/uif/src/reducer/handlers/handleUpdateSucceeded.ts @@ -9,24 +9,21 @@ export function handleUpdateSucceeded( action: UpdateSucceededAction, ): IndexerReducerResult { assertStatus(state.status, 'updating') - if (action.targetHeight >= state.height) { + if (action.newHeight >= state.height) { state = { ...state, status: 'idle', - height: action.targetHeight, + height: action.newHeight, invalidateToHeight: state.invalidateToHeight === state.height && !state.forceInvalidate - ? action.targetHeight + ? action.newHeight : state.invalidateToHeight, } } else { state = { ...state, status: 'idle', - invalidateToHeight: Math.min( - action.targetHeight, - state.invalidateToHeight, - ), + invalidateToHeight: Math.min(action.newHeight, state.invalidateToHeight), forceInvalidate: true, } } diff --git a/packages/uif/src/reducer/indexerReducer.test.ts b/packages/uif/src/reducer/indexerReducer.test.ts index 108d8bf1..48f7812b 100644 --- a/packages/uif/src/reducer/indexerReducer.test.ts +++ b/packages/uif/src/reducer/indexerReducer.test.ts @@ -244,7 +244,7 @@ describe(indexerReducer.name, () => { expect(effects1).toEqual([{ type: 'Update', targetHeight: 150 }]) const [state2, effects2] = reduceWithIndexerReducer(state1, [ - { type: 'UpdateSucceeded', from: 100, targetHeight: 150 }, + { type: 'UpdateSucceeded', from: 100, newHeight: 150 }, ]) expect(state2).toEqual({ @@ -415,7 +415,7 @@ describe(indexerReducer.name, () => { const [state2, effects2] = reduceWithIndexerReducer(state, [ { type: 'ParentUpdated', index: 0, safeHeight: 50 }, - { type: 'UpdateSucceeded', from: 100, targetHeight: 200 }, + { type: 'UpdateSucceeded', from: 100, newHeight: 200 }, ]) expect(state2).toEqual({ @@ -470,7 +470,7 @@ describe(indexerReducer.name, () => { const [state1, effects1] = reduceWithIndexerReducer(initState, [ { type: 'ParentUpdated', index: 0, safeHeight: 200 }, - { type: 'UpdateSucceeded', from: 100, targetHeight: 150 }, + { type: 'UpdateSucceeded', from: 100, newHeight: 150 }, ]) expect(effects1).toEqual([ @@ -485,7 +485,7 @@ describe(indexerReducer.name, () => { expect(effects2).toEqual([{ type: 'SetSafeHeight', safeHeight: 140 }]) const [, effects3] = reduceWithIndexerReducer(state2, [ - { type: 'UpdateSucceeded', from: 150, targetHeight: 200 }, + { type: 'UpdateSucceeded', from: 150, newHeight: 200 }, ]) expect(effects3).toEqual([ @@ -493,6 +493,7 @@ describe(indexerReducer.name, () => { { type: 'Invalidate', targetHeight: 140 }, ]) }) + it('if partially invalidating, does not update until done invalidating fully', () => { const initState = getAfterInit({ safeHeight: 100, @@ -625,6 +626,27 @@ describe(indexerReducer.name, () => { { type: 'Tick' }, ]) }) + + it('does not save the same safe height', () => { + const initState = getInitialState(0) + const [state, effects] = reduceWithIndexerReducer(initState, [ + { type: 'Initialized', safeHeight: 100, childCount: 0 }, + { type: 'RequestTick' }, + { type: 'TickSucceeded', safeHeight: 100 }, + { type: 'RequestTick' }, + { type: 'TickSucceeded', safeHeight: 100 }, + ]) + + expect(state).toEqual({ + ...initState, + status: 'idle', + height: 100, + safeHeight: 100, + invalidateToHeight: 100, + initializedSelf: true, + }) + expect(effects).toEqual([]) + }) }) describe('fatal errors', () => { @@ -841,7 +863,7 @@ describe(indexerReducer.name, () => { expect(effects3).toEqual([{ type: 'Update', targetHeight: 300 }]) const [state4, effects4] = reduceWithIndexerReducer(state3, [ - { type: 'UpdateSucceeded', from: 100, targetHeight: 300 }, + { type: 'UpdateSucceeded', from: 100, newHeight: 300 }, ]) // continues update as usual diff --git a/packages/uif/src/reducer/types/IndexerAction.ts b/packages/uif/src/reducer/types/IndexerAction.ts index 7ce14f60..43f5bd08 100644 --- a/packages/uif/src/reducer/types/IndexerAction.ts +++ b/packages/uif/src/reducer/types/IndexerAction.ts @@ -18,7 +18,7 @@ export interface ChildReadyAction { export interface UpdateSucceededAction { type: 'UpdateSucceeded' from: number - targetHeight: number + newHeight: number } export interface UpdateFailedAction { diff --git a/packages/uif/tsconfig.example.json b/packages/uif/tsconfig.example.json deleted file mode 100644 index 7367ec84..00000000 --- a/packages/uif/tsconfig.example.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extends": "./tsconfig.json", - "compilerOptions": { - "noEmit": true - }, - "include": ["example"] -}