From e37b341cd3ed46387905a75359539ec45b1b34f8 Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Wed, 20 Mar 2024 15:40:38 +0100 Subject: [PATCH 01/31] Add support for undefined height --- .../uif/example/BlockNumberIndexer.example.ts | 12 +- .../example/ConfigurableIndexer.example.ts | 11 +- packages/uif/example/HourlyIndexer.example.ts | 24 ++-- packages/uif/src/BaseIndexer.test.ts | 8 +- packages/uif/src/BaseIndexer.ts | 135 +++++++++++------- packages/uif/src/Indexer.ts | 2 +- packages/uif/src/SliceIndexer.ts | 2 +- packages/uif/src/height.ts | 51 +++++++ .../reducer/handlers/handleParentUpdated.ts | 4 +- .../reducer/handlers/handleUpdateSucceeded.ts | 11 +- .../src/reducer/helpers/continueOperations.ts | 21 ++- .../reducer/helpers/finishInitialization.ts | 5 +- .../uif/src/reducer/indexerReducer.test.ts | 10 +- .../uif/src/reducer/types/IndexerAction.ts | 10 +- .../uif/src/reducer/types/IndexerEffect.ts | 4 +- .../uif/src/reducer/types/IndexerState.ts | 8 +- 16 files changed, 203 insertions(+), 115 deletions(-) create mode 100644 packages/uif/src/height.ts diff --git a/packages/uif/example/BlockNumberIndexer.example.ts b/packages/uif/example/BlockNumberIndexer.example.ts index 8fe592f2..44e66068 100644 --- a/packages/uif/example/BlockNumberIndexer.example.ts +++ b/packages/uif/example/BlockNumberIndexer.example.ts @@ -52,9 +52,9 @@ interface IndexerRepository { } export class ClockIndexer extends RootIndexer { - override async start(): Promise { - await super.start() + override async initialize(): Promise { setInterval(() => this.requestTick(), 4 * 1000) + return this.tick() } async tick(): Promise { @@ -82,9 +82,9 @@ export class BlockDownloader extends ChildIndexer { this.id = 'BlockDownloader' // this should be unique across all indexers } - override async start(): Promise { - await super.start() + override async initialize(): Promise { this.lastKnownNumber = (await this.blockRepository.findLast())?.number ?? 0 + return this.indexerRepository.getSafeHeight(this.id) } protected async update( @@ -155,10 +155,6 @@ export class BlockDownloader extends ChildIndexer { await this.indexerRepository.setSafeHeight(this.id, height) } - getSafeHeight(): Promise { - return this.indexerRepository.getSafeHeight(this.id) - } - private async getKnownBlock(blockNumber: number): Promise { const known = await this.blockRepository.findByNumber(blockNumber) if (known) { diff --git a/packages/uif/example/ConfigurableIndexer.example.ts b/packages/uif/example/ConfigurableIndexer.example.ts index b1f23243..59c351a8 100644 --- a/packages/uif/example/ConfigurableIndexer.example.ts +++ b/packages/uif/example/ConfigurableIndexer.example.ts @@ -30,18 +30,18 @@ export class ConfigurableIndexer extends ChildIndexer { super(logger, [parentIndexer]) } - override async start(): Promise { + override async initialize(): Promise { const oldConfigHash = await this.stateRepository.getConfigHash() const newConfigHash = this.config.getConfigHash() if (oldConfigHash !== newConfigHash) { await this.stateRepository.setSafeHeight(0) await this.stateRepository.setConfigHash(newConfigHash) } - await super.start() + return this.stateRepository.getSafeHeight() } override async update(from: number, to: number): Promise { - const data = [] + const data: number[] = [] for (let i = from + 1; i <= to; i++) { data.push(i) } @@ -55,11 +55,6 @@ export class ConfigurableIndexer extends ChildIndexer { return targetHeight } - override async getSafeHeight(): Promise { - const height = await this.stateRepository.getSafeHeight() - return height - } - override async setSafeHeight(height: number): Promise { return this.stateRepository.setSafeHeight(height) } diff --git a/packages/uif/example/HourlyIndexer.example.ts b/packages/uif/example/HourlyIndexer.example.ts index eb2fceb0..2197bfe6 100644 --- a/packages/uif/example/HourlyIndexer.example.ts +++ b/packages/uif/example/HourlyIndexer.example.ts @@ -1,21 +1,19 @@ import { RootIndexer } from '../src/BaseIndexer' -const MS_IN_HOUR = 60 * 60 * 1000 - export class HourlyIndexer extends RootIndexer { - override async start(): Promise { - await super.start() - setInterval(() => this.requestTick(), MS_IN_HOUR) + // initialize is only called once when the indexer is started + async initialize() { + // check every second + setInterval(() => this.requestTick(), 1000) + return this.tick() } + // tick is called every time we request a new tick + // It should return the new target height tick(): Promise { - const time = getLastFullHourTimestampSeconds() - return Promise.resolve(time) + const ONE_HOUR = 60 * 60 * 1000 + const now = Date.now() + const lastHour = now - (now % ONE_HOUR) + return Promise.resolve(lastHour) } } - -function getLastFullHourTimestampSeconds(): number { - const now = Date.now() - const lastFullHour = now - (now % MS_IN_HOUR) - return Math.floor(lastFullHour / 1000) -} diff --git a/packages/uif/src/BaseIndexer.test.ts b/packages/uif/src/BaseIndexer.test.ts index 585db584..2dbce739 100644 --- a/packages/uif/src/BaseIndexer.test.ts +++ b/packages/uif/src/BaseIndexer.test.ts @@ -21,7 +21,7 @@ describe(BaseIndexer.name, () => { await child.finishUpdate(1) - expect(await child.getSafeHeight()).toEqual(1) + expect(await child.initialize()).toEqual(1) }) it('first parent update then invalidate', async () => { @@ -37,7 +37,7 @@ describe(BaseIndexer.name, () => { await child.finishUpdate(1) - expect(await child.getSafeHeight()).toEqual(1) + expect(await child.initialize()).toEqual(1) }) }) @@ -276,7 +276,7 @@ export class TestRootIndexer extends RootIndexer { }) } - override async getSafeHeight(): Promise { + override async initialize(): Promise { const promise = this.tick() this.resolveTick(this.safeHeight) await promise @@ -339,7 +339,7 @@ class TestChildIndexer extends ChildIndexer { }) } - override getSafeHeight(): Promise { + override initialize(): Promise { return Promise.resolve(this.safeHeight) } diff --git a/packages/uif/src/BaseIndexer.ts b/packages/uif/src/BaseIndexer.ts index f8bf88d0..e6e6a07f 100644 --- a/packages/uif/src/BaseIndexer.ts +++ b/packages/uif/src/BaseIndexer.ts @@ -3,6 +3,7 @@ import { assert } from 'node:console' import { Logger } from '@l2beat/backend-tools' import { assertUnreachable } from './assertUnreachable' +import { Height } from './height' import { Indexer } from './Indexer' import { getInitialState } from './reducer/getInitialState' import { indexerReducer } from './reducer/indexerReducer' @@ -21,8 +22,8 @@ export abstract class BaseIndexer implements Indexer { /** * This can be overridden to provide a custom retry strategy. It will be - * used for all indexers in the app that don't specify their own strategy. - * @returns A default retry strategy that will be used for all indexers in the app + * used for all indexers that don't specify their own strategy. + * @returns A default retry strategy that will be used for all indexers */ static GET_DEFAULT_RETRY_STRATEGY: () => RetryStrategy = () => Retries.exponentialBackOff({ @@ -32,37 +33,93 @@ export abstract class BaseIndexer implements Indexer { }) /** - * Should read the height from the database. It must return a height, so - * if database is empty a fallback value has to be chosen. + * Initializes the indexer. It should return a height that the indexer has + * synced up to. If the indexer has not synced any data, it should return + * undefined. + * + * This method is expected to read the height that was saved previously with + * `setSafeHeight`. It shouldn't call `setSafeHeight` itself. + * + * For root indexers this method should also schedule a process to request + * ticks. For example with `setInterval(() => this.requestTick(), 1000)`. + * Since a root indexer probably doesn't save the height to a database, it + * can `return this.tick()` instead. */ - abstract getSafeHeight(): Promise + abstract initialize(): Promise /** - * Should write the height to the database. The height given is the most - * pessimistic value and the indexer is expected to actually operate at a - * higher height in runtime. + * Saves the height (most likely to a database). The height given is the + * smallest height from all parents and what the indexer itself synced to + * previously. It can be undefined. + * + * When `initialize` is called it is expected that it will read the same + * height that was saved here. */ - protected abstract setSafeHeight(height: number): Promise + protected abstract setSafeHeight(height: number | undefined): Promise /** - * To be used in ChildIndexer. + * This method should only be implemented for a child indexer. + * + * It is responsible for the main data fetching process. It is up to the + * indexer to decide how much data to fetch. For example given + * `.update(100, 200)`, the indexer can only fetch data up to 110 and return + * 110. The next time this method will be called with `.update(110, 200)`. * - * @param from - current height of the indexer, exclusive - * @param to - inclusive + * @param from The height that the indexer has synced up to previously. Can + * be undefined if no data was synced. This value is inclusive so the indexer + * should not fetch data for this height. + * + * @param to The height that the indexer should sync up to. This value is + * exclusive so the indexer should fetch data for this height. + * + * @returns The height that the indexer has synced up to. Returning `from` + * means that the indexer has not synced any data. Returning a value greater + * than `from` means that the indexer has synced up to that height. Returning + * a value less than `from` will trigger invalidation down to the returned + * value. Returning `undefined` will invalidate all data. Returning a value + * greater than `to` is not permitted. */ - // TODO: do we need to pass the current height? - protected abstract update(from: number, to: number): Promise + protected abstract update( + from: number | undefined, + to: number, + ): Promise /** - * Only to be used in RootIndexer. It provides a way to start the height - * update process. + * This method should only be implemented for a child indexer. + * + * It is responsible for invalidating data that was synced previously. It is + * possible that no data was synced and this method is still called. + * + * Invalidation can, but doesn't have to remove data from the database. If + * you only want to rely on the safe height, you can just return the target + * height and the system will take care of the rest. + * + * This method doesn't have to invalidate all data. If you want to do it in + * steps, you can return a height that is larger than the target height. + * + * @param targetHeight The height that the indexer should invalidate down to. + * Can be undefined. If it is undefined, the indexer should invalidate all + * data. + * + * @returns The height that the indexer has invalidated down to. Returning + * `targetHeight` means that the indexer has invalidated all the required + * data. Returning a value greater than `targetHeight` means that the indexer + * has invalidated down to that height. */ - protected abstract tick(): Promise + protected abstract invalidate( + targetHeight: number | undefined, + ): Promise /** - * @param targetHeight - every value > `targetHeight` is invalid, but `targetHeight` itself is valid + * This method should only be implemented for a root indexer. + * + * It is responsible for providing the target height for the entire system. + * Some good examples of this are: the current time or the last block number. + * + * As opposed to `update` and `invalidate`, this method cannot return + * `undefined`. */ - protected abstract invalidate(targetHeight: number): Promise + protected abstract tick(): Promise private state: IndexerState private started = false @@ -99,7 +156,7 @@ export abstract class BaseIndexer implements Indexer { async start(): Promise { assert(!this.started, 'Indexer already started') this.started = true - const height = await this.getSafeHeight() + const height = await this.initialize() this.dispatch({ type: 'Initialized', safeHeight: height, @@ -120,13 +177,13 @@ export abstract class BaseIndexer implements Indexer { this.dispatch({ type: 'ChildReady', index }) } - notifyUpdate(parent: Indexer, to: number): void { + notifyUpdate(parent: Indexer, safeHeight: number | undefined): void { this.logger.debug('Someone has updated', { parent: parent.constructor.name, }) const index = this.parents.indexOf(parent) assert(index !== -1, 'Received update from unknown parent') - this.dispatch({ type: 'ParentUpdated', index, safeHeight: to }) + this.dispatch({ type: 'ParentUpdated', index, safeHeight }) } getState(): IndexerState { @@ -167,19 +224,18 @@ export abstract class BaseIndexer implements Indexer { // #region Child methods private async executeUpdate(effect: UpdateEffect): Promise { - // TODO: maybe from should be inclusive? const from = this.state.height this.logger.info('Updating', { from, to: effect.targetHeight }) try { - const to = await this.update(from, effect.targetHeight) - if (to > effect.targetHeight) { + const newHeight = await this.update(from, effect.targetHeight) + if (Height.gt(newHeight, effect.targetHeight)) { this.logger.critical('Update returned invalid height', { - returned: to, + newHeight, max: effect.targetHeight, }) this.dispatch({ type: 'UpdateFailed', fatal: true }) } else { - this.dispatch({ type: 'UpdateSucceeded', from, targetHeight: to }) + this.dispatch({ type: 'UpdateSucceeded', from, newHeight }) this.updateRetryStrategy.clear() } } catch (e) { @@ -299,18 +355,12 @@ export abstract class RootIndexer extends BaseIndexer { super(logger, [], opts) } - // eslint-disable-next-line @typescript-eslint/require-await override async update(): Promise { - throw new Error('RootIndexer cannot update') + return Promise.reject(new Error('RootIndexer cannot update')) } - // eslint-disable-next-line @typescript-eslint/require-await override async invalidate(): Promise { - throw new Error('RootIndexer cannot invalidate') - } - - override async getSafeHeight(): Promise { - return this.tick() + return Promise.reject(new Error('RootIndexer cannot invalidate')) } override async setSafeHeight(): Promise { @@ -319,20 +369,7 @@ export abstract class RootIndexer extends BaseIndexer { } export abstract class ChildIndexer extends BaseIndexer { - // eslint-disable-next-line @typescript-eslint/no-useless-constructor - constructor( - logger: Logger, - parents: Indexer[], - opts?: { - updateRetryStrategy?: RetryStrategy - invalidateRetryStrategy?: RetryStrategy - }, - ) { - super(logger, parents, opts) - } - - // eslint-disable-next-line @typescript-eslint/require-await override async tick(): Promise { - throw new Error('ChildIndexer cannot tick') + return Promise.reject(new Error('ChildIndexer cannot tick')) } } diff --git a/packages/uif/src/Indexer.ts b/packages/uif/src/Indexer.ts index 6b23c216..adf86cdb 100644 --- a/packages/uif/src/Indexer.ts +++ b/packages/uif/src/Indexer.ts @@ -6,6 +6,6 @@ export interface UpdateEvent { export interface Indexer { subscribe(child: Indexer): void notifyReady(child: Indexer): void - notifyUpdate(parent: Indexer, safeHeight: number): void + notifyUpdate(parent: Indexer, safeHeight: number | undefined): void start(): Promise } diff --git a/packages/uif/src/SliceIndexer.ts b/packages/uif/src/SliceIndexer.ts index 7992f38e..88f06f56 100644 --- a/packages/uif/src/SliceIndexer.ts +++ b/packages/uif/src/SliceIndexer.ts @@ -44,7 +44,7 @@ export abstract class SliceIndexer extends ChildIndexer { return to } - override async getSafeHeight(): Promise { + override async initialize(): Promise { const sliceStates = await this.getSliceStates() const mainSafeHeight = await this.getMainSafeHeight() return Math.min(...sliceStates.map((s) => s.height), mainSafeHeight) diff --git a/packages/uif/src/height.ts b/packages/uif/src/height.ts new file mode 100644 index 00000000..95a83667 --- /dev/null +++ b/packages/uif/src/height.ts @@ -0,0 +1,51 @@ +export const Height = { + lt, + lte, + gt, + gte, + min, +} + +function lt(heightA: number | undefined, heightB: number | undefined): boolean { + if (heightA === heightB) { + return false + } + return !gt(heightA, heightB) +} + +function lte( + heightA: number | undefined, + heightB: number | undefined, +): boolean { + return !gt(heightA, heightB) +} + +function gt(heightA: number | undefined, heightB: number | undefined): boolean { + if (heightA === undefined) { + return false + } + if (heightB === undefined) { + return true + } + return heightA > heightB +} + +function gte( + heightA: number | undefined, + heightB: number | undefined, +): boolean { + return !lt(heightA, heightB) +} + +function min(...heights: (number | undefined)[]): number | undefined { + if (heights.length === 0) { + return undefined + } + let minHeight = heights[0] + for (const height of heights) { + if (gt(minHeight, height)) { + minHeight = height + } + } + return minHeight +} diff --git a/packages/uif/src/reducer/handlers/handleParentUpdated.ts b/packages/uif/src/reducer/handlers/handleParentUpdated.ts index ac02ddba..d308eb0a 100644 --- a/packages/uif/src/reducer/handlers/handleParentUpdated.ts +++ b/packages/uif/src/reducer/handlers/handleParentUpdated.ts @@ -1,3 +1,4 @@ +import { Height } from '../../height' import { continueOperations } from '../helpers/continueOperations' import { finishInitialization } from '../helpers/finishInitialization' import { ParentUpdatedAction } from '../types/IndexerAction' @@ -12,7 +13,8 @@ export function handleParentUpdated( ...state, parents: state.parents.map((parent, index) => { if (index === action.index) { - const waiting = parent.waiting || action.safeHeight < parent.safeHeight + const waiting = + parent.waiting || Height.lt(action.safeHeight, parent.safeHeight) return { ...parent, safeHeight: action.safeHeight, diff --git a/packages/uif/src/reducer/handlers/handleUpdateSucceeded.ts b/packages/uif/src/reducer/handlers/handleUpdateSucceeded.ts index 39ea011a..95dbb269 100644 --- a/packages/uif/src/reducer/handlers/handleUpdateSucceeded.ts +++ b/packages/uif/src/reducer/handlers/handleUpdateSucceeded.ts @@ -1,3 +1,4 @@ +import { Height } from '../../height' import { assertStatus } from '../helpers/assertStatus' import { continueOperations } from '../helpers/continueOperations' import { UpdateSucceededAction } from '../types/IndexerAction' @@ -9,22 +10,22 @@ export function handleUpdateSucceeded( action: UpdateSucceededAction, ): IndexerReducerResult { assertStatus(state.status, 'updating') - if (action.targetHeight >= state.height) { + if (Height.gte(action.newHeight, state.height)) { state = { ...state, status: 'idle', - height: action.targetHeight, + height: action.newHeight, invalidateToHeight: state.invalidateToHeight === state.height && !state.forceInvalidate - ? action.targetHeight + ? action.newHeight : state.invalidateToHeight, } } else { state = { ...state, status: 'idle', - invalidateToHeight: Math.min( - action.targetHeight, + invalidateToHeight: Height.min( + action.newHeight, state.invalidateToHeight, ), forceInvalidate: true, diff --git a/packages/uif/src/reducer/helpers/continueOperations.ts b/packages/uif/src/reducer/helpers/continueOperations.ts index 9dc85053..372ac82e 100644 --- a/packages/uif/src/reducer/helpers/continueOperations.ts +++ b/packages/uif/src/reducer/helpers/continueOperations.ts @@ -1,5 +1,6 @@ import assert from 'node:assert' +import { Height } from '../../height' import { IndexerEffect } from '../types/IndexerEffect' import { IndexerReducerResult } from '../types/IndexerReducerResult' import { IndexerState } from '../types/IndexerState' @@ -13,25 +14,30 @@ export function continueOperations( } = {}, ): IndexerReducerResult { const initializedParents = state.parents.filter((x) => x.initialized) - const parentHeight = Math.min(...initializedParents.map((x) => x.safeHeight)) + const parentHeight = Height.min( + ...initializedParents.map((x) => x.safeHeight), + ) if (initializedParents.length > 0) { state = { ...state, - invalidateToHeight: Math.min(state.invalidateToHeight, parentHeight), + invalidateToHeight: Height.min(state.invalidateToHeight, parentHeight), } } const effects: IndexerEffect[] = [] - if (state.invalidateToHeight < state.safeHeight || options.updateFinished) { - const safeHeight = Math.min(state.invalidateToHeight, state.height) + if ( + Height.lt(state.invalidateToHeight, state.safeHeight) || + options.updateFinished + ) { + const safeHeight = Height.min(state.invalidateToHeight, state.height) if (safeHeight !== state.safeHeight) { effects.push({ type: 'SetSafeHeight', safeHeight }) } - if (safeHeight < state.safeHeight) { + if (Height.lt(safeHeight, state.safeHeight)) { state = { ...state, safeHeight, @@ -85,11 +91,12 @@ export function continueOperations( } const shouldInvalidate = - state.forceInvalidate || state.invalidateToHeight < state.height + state.forceInvalidate || Height.lt(state.invalidateToHeight, state.height) const shouldUpdate = !shouldInvalidate && initializedParents.length > 0 && - parentHeight > state.height + Height.gt(parentHeight, state.height) && + parentHeight !== undefined if (shouldInvalidate) { if (state.invalidateBlocked || state.waiting || state.status !== 'idle') { diff --git a/packages/uif/src/reducer/helpers/finishInitialization.ts b/packages/uif/src/reducer/helpers/finishInitialization.ts index b749de4f..7e2b10aa 100644 --- a/packages/uif/src/reducer/helpers/finishInitialization.ts +++ b/packages/uif/src/reducer/helpers/finishInitialization.ts @@ -1,3 +1,4 @@ +import { Height } from '../../height' import { IndexerReducerResult } from '../types/IndexerReducerResult' import { IndexerState } from '../types/IndexerState' @@ -21,8 +22,8 @@ export function finishInitialization( } if (state.parents.every((x) => x.initialized)) { - const parentHeight = Math.min(...state.parents.map((x) => x.safeHeight)) - const height = Math.min(parentHeight, state.height) + const parentHeight = Height.min(...state.parents.map((x) => x.safeHeight)) + const height = Height.min(parentHeight, state.height) return [ { diff --git a/packages/uif/src/reducer/indexerReducer.test.ts b/packages/uif/src/reducer/indexerReducer.test.ts index 108d8bf1..e5506b63 100644 --- a/packages/uif/src/reducer/indexerReducer.test.ts +++ b/packages/uif/src/reducer/indexerReducer.test.ts @@ -244,7 +244,7 @@ describe(indexerReducer.name, () => { expect(effects1).toEqual([{ type: 'Update', targetHeight: 150 }]) const [state2, effects2] = reduceWithIndexerReducer(state1, [ - { type: 'UpdateSucceeded', from: 100, targetHeight: 150 }, + { type: 'UpdateSucceeded', from: 100, newHeight: 150 }, ]) expect(state2).toEqual({ @@ -415,7 +415,7 @@ describe(indexerReducer.name, () => { const [state2, effects2] = reduceWithIndexerReducer(state, [ { type: 'ParentUpdated', index: 0, safeHeight: 50 }, - { type: 'UpdateSucceeded', from: 100, targetHeight: 200 }, + { type: 'UpdateSucceeded', from: 100, newHeight: 200 }, ]) expect(state2).toEqual({ @@ -470,7 +470,7 @@ describe(indexerReducer.name, () => { const [state1, effects1] = reduceWithIndexerReducer(initState, [ { type: 'ParentUpdated', index: 0, safeHeight: 200 }, - { type: 'UpdateSucceeded', from: 100, targetHeight: 150 }, + { type: 'UpdateSucceeded', from: 100, newHeight: 150 }, ]) expect(effects1).toEqual([ @@ -485,7 +485,7 @@ describe(indexerReducer.name, () => { expect(effects2).toEqual([{ type: 'SetSafeHeight', safeHeight: 140 }]) const [, effects3] = reduceWithIndexerReducer(state2, [ - { type: 'UpdateSucceeded', from: 150, targetHeight: 200 }, + { type: 'UpdateSucceeded', from: 150, newHeight: 200 }, ]) expect(effects3).toEqual([ @@ -841,7 +841,7 @@ describe(indexerReducer.name, () => { expect(effects3).toEqual([{ type: 'Update', targetHeight: 300 }]) const [state4, effects4] = reduceWithIndexerReducer(state3, [ - { type: 'UpdateSucceeded', from: 100, targetHeight: 300 }, + { type: 'UpdateSucceeded', from: 100, newHeight: 300 }, ]) // continues update as usual diff --git a/packages/uif/src/reducer/types/IndexerAction.ts b/packages/uif/src/reducer/types/IndexerAction.ts index 7ce14f60..7807bc8a 100644 --- a/packages/uif/src/reducer/types/IndexerAction.ts +++ b/packages/uif/src/reducer/types/IndexerAction.ts @@ -1,13 +1,13 @@ export interface InitializedAction { type: 'Initialized' - safeHeight: number + safeHeight: number | undefined childCount: number } export interface ParentUpdatedAction { type: 'ParentUpdated' index: number - safeHeight: number + safeHeight: number | undefined } export interface ChildReadyAction { @@ -17,8 +17,8 @@ export interface ChildReadyAction { export interface UpdateSucceededAction { type: 'UpdateSucceeded' - from: number - targetHeight: number + from: number | undefined + newHeight: number | undefined } export interface UpdateFailedAction { @@ -32,7 +32,7 @@ export interface RetryUpdateAction { export interface InvalidateSucceededAction { type: 'InvalidateSucceeded' - targetHeight: number + targetHeight: number | undefined } export interface InvalidateFailedAction { diff --git a/packages/uif/src/reducer/types/IndexerEffect.ts b/packages/uif/src/reducer/types/IndexerEffect.ts index 5a7c0d34..c93c4400 100644 --- a/packages/uif/src/reducer/types/IndexerEffect.ts +++ b/packages/uif/src/reducer/types/IndexerEffect.ts @@ -15,12 +15,12 @@ export interface UpdateEffect { export interface InvalidateEffect { type: 'Invalidate' - targetHeight: number + targetHeight: number | undefined } export interface SetSafeHeightEffect { type: 'SetSafeHeight' - safeHeight: number + safeHeight: number | undefined } export interface NotifyReadyEffect { diff --git a/packages/uif/src/reducer/types/IndexerState.ts b/packages/uif/src/reducer/types/IndexerState.ts index 5ac64e68..e3a64edf 100644 --- a/packages/uif/src/reducer/types/IndexerState.ts +++ b/packages/uif/src/reducer/types/IndexerState.ts @@ -6,12 +6,12 @@ export interface IndexerState { | 'invalidating' | 'ticking' | 'errored' - readonly height: number - readonly invalidateToHeight: number + readonly height: number | undefined + readonly invalidateToHeight: number | undefined readonly forceInvalidate: boolean // When we change safe height to a lower value we become waiting // and we mark all children as not ready - readonly safeHeight: number + readonly safeHeight: number | undefined readonly waiting: boolean readonly tickScheduled: boolean readonly initializedSelf: boolean @@ -19,7 +19,7 @@ export interface IndexerState { readonly initialized: boolean // When the parent changes safeHeight to a lower value // we mark them as waiting and will notify them when we're ready - readonly safeHeight: number + readonly safeHeight: number | undefined readonly waiting: boolean }[] readonly children: { From b4b4e7be92efb314e61a6db8dc5264bec05fa1e7 Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Thu, 21 Mar 2024 09:23:25 +0100 Subject: [PATCH 02/31] Refactor undefined to null for height --- packages/uif/src/BaseIndexer.ts | 28 +++++++++---------- packages/uif/src/Indexer.ts | 2 +- packages/uif/src/height.ts | 24 ++++++---------- .../src/reducer/helpers/continueOperations.ts | 2 +- .../uif/src/reducer/types/IndexerAction.ts | 10 +++---- .../uif/src/reducer/types/IndexerEffect.ts | 4 +-- .../uif/src/reducer/types/IndexerState.ts | 8 +++--- 7 files changed, 36 insertions(+), 42 deletions(-) diff --git a/packages/uif/src/BaseIndexer.ts b/packages/uif/src/BaseIndexer.ts index e6e6a07f..d2306fe4 100644 --- a/packages/uif/src/BaseIndexer.ts +++ b/packages/uif/src/BaseIndexer.ts @@ -35,7 +35,7 @@ export abstract class BaseIndexer implements Indexer { /** * Initializes the indexer. It should return a height that the indexer has * synced up to. If the indexer has not synced any data, it should return - * undefined. + * `null`. * * This method is expected to read the height that was saved previously with * `setSafeHeight`. It shouldn't call `setSafeHeight` itself. @@ -45,17 +45,17 @@ export abstract class BaseIndexer implements Indexer { * Since a root indexer probably doesn't save the height to a database, it * can `return this.tick()` instead. */ - abstract initialize(): Promise + protected abstract initialize(): Promise /** * Saves the height (most likely to a database). The height given is the * smallest height from all parents and what the indexer itself synced to - * previously. It can be undefined. + * previously. It can be `null`. * * When `initialize` is called it is expected that it will read the same * height that was saved here. */ - protected abstract setSafeHeight(height: number | undefined): Promise + protected abstract setSafeHeight(height: number | null): Promise /** * This method should only be implemented for a child indexer. @@ -66,23 +66,23 @@ export abstract class BaseIndexer implements Indexer { * 110. The next time this method will be called with `.update(110, 200)`. * * @param from The height that the indexer has synced up to previously. Can - * be undefined if no data was synced. This value is inclusive so the indexer + * be `null` if no data was synced. This value is exclusive so the indexer * should not fetch data for this height. * * @param to The height that the indexer should sync up to. This value is - * exclusive so the indexer should fetch data for this height. + * inclusive so the indexer should fetch data for this height. * * @returns The height that the indexer has synced up to. Returning `from` * means that the indexer has not synced any data. Returning a value greater * than `from` means that the indexer has synced up to that height. Returning * a value less than `from` will trigger invalidation down to the returned - * value. Returning `undefined` will invalidate all data. Returning a value + * value. Returning `null` will invalidate all data. Returning a value * greater than `to` is not permitted. */ protected abstract update( - from: number | undefined, + from: number | null, to: number, - ): Promise + ): Promise /** * This method should only be implemented for a child indexer. @@ -98,7 +98,7 @@ export abstract class BaseIndexer implements Indexer { * steps, you can return a height that is larger than the target height. * * @param targetHeight The height that the indexer should invalidate down to. - * Can be undefined. If it is undefined, the indexer should invalidate all + * Can be `null`. If it is `null`, the indexer should invalidate all * data. * * @returns The height that the indexer has invalidated down to. Returning @@ -107,8 +107,8 @@ export abstract class BaseIndexer implements Indexer { * has invalidated down to that height. */ protected abstract invalidate( - targetHeight: number | undefined, - ): Promise + targetHeight: number | null, + ): Promise /** * This method should only be implemented for a root indexer. @@ -117,7 +117,7 @@ export abstract class BaseIndexer implements Indexer { * Some good examples of this are: the current time or the last block number. * * As opposed to `update` and `invalidate`, this method cannot return - * `undefined`. + * `null`. */ protected abstract tick(): Promise @@ -177,7 +177,7 @@ export abstract class BaseIndexer implements Indexer { this.dispatch({ type: 'ChildReady', index }) } - notifyUpdate(parent: Indexer, safeHeight: number | undefined): void { + notifyUpdate(parent: Indexer, safeHeight: number | null): void { this.logger.debug('Someone has updated', { parent: parent.constructor.name, }) diff --git a/packages/uif/src/Indexer.ts b/packages/uif/src/Indexer.ts index adf86cdb..bd5d7a26 100644 --- a/packages/uif/src/Indexer.ts +++ b/packages/uif/src/Indexer.ts @@ -6,6 +6,6 @@ export interface UpdateEvent { export interface Indexer { subscribe(child: Indexer): void notifyReady(child: Indexer): void - notifyUpdate(parent: Indexer, safeHeight: number | undefined): void + notifyUpdate(parent: Indexer, safeHeight: number | null): void start(): Promise } diff --git a/packages/uif/src/height.ts b/packages/uif/src/height.ts index 95a83667..705e53b0 100644 --- a/packages/uif/src/height.ts +++ b/packages/uif/src/height.ts @@ -6,42 +6,36 @@ export const Height = { min, } -function lt(heightA: number | undefined, heightB: number | undefined): boolean { +function lt(heightA: number | null, heightB: number | null): boolean { if (heightA === heightB) { return false } return !gt(heightA, heightB) } -function lte( - heightA: number | undefined, - heightB: number | undefined, -): boolean { +function lte(heightA: number | null, heightB: number | null): boolean { return !gt(heightA, heightB) } -function gt(heightA: number | undefined, heightB: number | undefined): boolean { - if (heightA === undefined) { +function gt(heightA: number | null, heightB: number | null): boolean { + if (heightA === null) { return false } - if (heightB === undefined) { + if (heightB === null) { return true } return heightA > heightB } -function gte( - heightA: number | undefined, - heightB: number | undefined, -): boolean { +function gte(heightA: number | null, heightB: number | null): boolean { return !lt(heightA, heightB) } -function min(...heights: (number | undefined)[]): number | undefined { +function min(...heights: (number | null)[]): number | null { if (heights.length === 0) { - return undefined + return null } - let minHeight = heights[0] + let minHeight = heights[0] ?? null for (const height of heights) { if (gt(minHeight, height)) { minHeight = height diff --git a/packages/uif/src/reducer/helpers/continueOperations.ts b/packages/uif/src/reducer/helpers/continueOperations.ts index 372ac82e..e04955ea 100644 --- a/packages/uif/src/reducer/helpers/continueOperations.ts +++ b/packages/uif/src/reducer/helpers/continueOperations.ts @@ -96,7 +96,7 @@ export function continueOperations( !shouldInvalidate && initializedParents.length > 0 && Height.gt(parentHeight, state.height) && - parentHeight !== undefined + parentHeight !== null if (shouldInvalidate) { if (state.invalidateBlocked || state.waiting || state.status !== 'idle') { diff --git a/packages/uif/src/reducer/types/IndexerAction.ts b/packages/uif/src/reducer/types/IndexerAction.ts index 7807bc8a..dd39ec54 100644 --- a/packages/uif/src/reducer/types/IndexerAction.ts +++ b/packages/uif/src/reducer/types/IndexerAction.ts @@ -1,13 +1,13 @@ export interface InitializedAction { type: 'Initialized' - safeHeight: number | undefined + safeHeight: number | null childCount: number } export interface ParentUpdatedAction { type: 'ParentUpdated' index: number - safeHeight: number | undefined + safeHeight: number | null } export interface ChildReadyAction { @@ -17,8 +17,8 @@ export interface ChildReadyAction { export interface UpdateSucceededAction { type: 'UpdateSucceeded' - from: number | undefined - newHeight: number | undefined + from: number | null + newHeight: number | null } export interface UpdateFailedAction { @@ -32,7 +32,7 @@ export interface RetryUpdateAction { export interface InvalidateSucceededAction { type: 'InvalidateSucceeded' - targetHeight: number | undefined + targetHeight: number | null } export interface InvalidateFailedAction { diff --git a/packages/uif/src/reducer/types/IndexerEffect.ts b/packages/uif/src/reducer/types/IndexerEffect.ts index c93c4400..8d04c31c 100644 --- a/packages/uif/src/reducer/types/IndexerEffect.ts +++ b/packages/uif/src/reducer/types/IndexerEffect.ts @@ -15,12 +15,12 @@ export interface UpdateEffect { export interface InvalidateEffect { type: 'Invalidate' - targetHeight: number | undefined + targetHeight: number | null } export interface SetSafeHeightEffect { type: 'SetSafeHeight' - safeHeight: number | undefined + safeHeight: number | null } export interface NotifyReadyEffect { diff --git a/packages/uif/src/reducer/types/IndexerState.ts b/packages/uif/src/reducer/types/IndexerState.ts index e3a64edf..12dd5976 100644 --- a/packages/uif/src/reducer/types/IndexerState.ts +++ b/packages/uif/src/reducer/types/IndexerState.ts @@ -6,12 +6,12 @@ export interface IndexerState { | 'invalidating' | 'ticking' | 'errored' - readonly height: number | undefined - readonly invalidateToHeight: number | undefined + readonly height: number | null + readonly invalidateToHeight: number | null readonly forceInvalidate: boolean // When we change safe height to a lower value we become waiting // and we mark all children as not ready - readonly safeHeight: number | undefined + readonly safeHeight: number | null readonly waiting: boolean readonly tickScheduled: boolean readonly initializedSelf: boolean @@ -19,7 +19,7 @@ export interface IndexerState { readonly initialized: boolean // When the parent changes safeHeight to a lower value // we mark them as waiting and will notify them when we're ready - readonly safeHeight: number | undefined + readonly safeHeight: number | null readonly waiting: boolean }[] readonly children: { From 023856f47a755277dcb0a1da2c1176963274b879 Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Thu, 21 Mar 2024 10:15:44 +0100 Subject: [PATCH 03/31] Rename update function parameters --- packages/uif/src/BaseIndexer.ts | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/uif/src/BaseIndexer.ts b/packages/uif/src/BaseIndexer.ts index d2306fe4..1d1e47eb 100644 --- a/packages/uif/src/BaseIndexer.ts +++ b/packages/uif/src/BaseIndexer.ts @@ -65,23 +65,23 @@ export abstract class BaseIndexer implements Indexer { * `.update(100, 200)`, the indexer can only fetch data up to 110 and return * 110. The next time this method will be called with `.update(110, 200)`. * - * @param from The height that the indexer has synced up to previously. Can + * @param currentHeight The height that the indexer has synced up to previously. Can * be `null` if no data was synced. This value is exclusive so the indexer * should not fetch data for this height. * - * @param to The height that the indexer should sync up to. This value is - * inclusive so the indexer should fetch data for this height. + * @param targetHeight The height that the indexer should sync up to. This value is + * inclusive so the indexer should eventually fetch data for this height. * - * @returns The height that the indexer has synced up to. Returning `from` - * means that the indexer has not synced any data. Returning a value greater - * than `from` means that the indexer has synced up to that height. Returning - * a value less than `from` will trigger invalidation down to the returned - * value. Returning `null` will invalidate all data. Returning a value - * greater than `to` is not permitted. + * @returns The height that the indexer has synced up to. Returning + * `currentHeight` means that the indexer has not synced any data. Returning + * a value greater than `currentHeight` means that the indexer has synced up + * to that height. Returning a value less than `currentHeight` will trigger + * invalidation down to the returned value. Returning `null` will invalidate + * all data. Returning a value greater than `targetHeight` is not permitted. */ protected abstract update( - from: number | null, - to: number, + currentHeight: number | null, + targetHeight: number, ): Promise /** From b1565b12a28a7668d43fb828d1477bd63830c91b Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Thu, 21 Mar 2024 12:02:01 +0100 Subject: [PATCH 04/31] Remove useless examples --- .../uif/example/BlockNumberIndexer.example.ts | 172 ------------------ .../example/ConfigurableIndexer.example.ts | 61 ------- packages/uif/example/HourlyIndexer.example.ts | 19 -- packages/uif/tsconfig.example.json | 7 - 4 files changed, 259 deletions(-) delete mode 100644 packages/uif/example/BlockNumberIndexer.example.ts delete mode 100644 packages/uif/example/ConfigurableIndexer.example.ts delete mode 100644 packages/uif/example/HourlyIndexer.example.ts delete mode 100644 packages/uif/tsconfig.example.json diff --git a/packages/uif/example/BlockNumberIndexer.example.ts b/packages/uif/example/BlockNumberIndexer.example.ts deleted file mode 100644 index 44e66068..00000000 --- a/packages/uif/example/BlockNumberIndexer.example.ts +++ /dev/null @@ -1,172 +0,0 @@ -import { Logger } from '@l2beat/backend-tools' - -import { ChildIndexer, RootIndexer } from '../src' - -interface Hash256 extends String { - _Hash256Brand: string -} - -function Hash256(hash: string): Hash256 { - return hash as unknown as Hash256 -} - -interface UnixTime extends Number { - _UnixTimeBrand: string -} - -function UnixTime(timestamp: number): UnixTime { - return timestamp as unknown as UnixTime -} - -interface BlockFromClient { - parentHash: string - hash: string - number: number - timestamp: number -} - -interface EthereumClient { - getBlock: (block: Hash256 | number) => Promise - getBlockByTimestamp: (timestamp: UnixTime) => Promise - getBlockNumber: () => Promise - onBlock: (handler: (blockNumber: number) => void) => () => void -} - -interface BlockRecord { - number: number - hash: Hash256 - timestamp: number -} - -interface BlockRepository { - findLast: () => Promise - findByNumber: (number: number) => Promise - findByTimestamp: (timestamp: UnixTime) => Promise - addMany: (blocks: BlockRecord[]) => Promise - deleteAfter: (block: number) => Promise -} - -interface IndexerRepository { - getSafeHeight: (indexerId: string) => Promise - setSafeHeight: (indexerId: string, height: number) => Promise -} - -export class ClockIndexer extends RootIndexer { - override async initialize(): Promise { - setInterval(() => this.requestTick(), 4 * 1000) - return this.tick() - } - - async tick(): Promise { - return Promise.resolve(getTimeSeconds()) - } -} - -function getTimeSeconds(): number { - return Math.floor(Date.now() / 1000) -} - -export class BlockDownloader extends ChildIndexer { - private lastKnownNumber = 0 - private reorgedBlocks = [] as BlockRecord[] - private readonly id: string - - constructor( - private readonly ethereumClient: EthereumClient, - private readonly blockRepository: BlockRepository, - private readonly indexerRepository: IndexerRepository, - clockIndexer: ClockIndexer, - logger: Logger, - ) { - super(logger, [clockIndexer]) - this.id = 'BlockDownloader' // this should be unique across all indexers - } - - override async initialize(): Promise { - this.lastKnownNumber = (await this.blockRepository.findLast())?.number ?? 0 - return this.indexerRepository.getSafeHeight(this.id) - } - - protected async update( - _fromTimestamp: number, - toTimestamp: number, - ): Promise { - if (this.reorgedBlocks.length > 0) { - // we do not need to check if lastKnown < to because we are sure that - // those blocks are from the past - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - this.lastKnownNumber = this.reorgedBlocks.at(-1)!.number - await this.blockRepository.addMany(this.reorgedBlocks) - this.reorgedBlocks = [] - } - - const tip = await this.ethereumClient.getBlockByTimestamp( - UnixTime(toTimestamp), - ) - if (tip.number <= this.lastKnownNumber) { - return tip.timestamp - } - - return await this.advanceChain(this.lastKnownNumber + 1) - } - - protected async invalidate(to: number): Promise { - await this.blockRepository.deleteAfter(to) - return to - } - - private async advanceChain(blockNumber: number): Promise { - let [block, parent] = await Promise.all([ - this.ethereumClient.getBlock(blockNumber), - this.getKnownBlock(blockNumber - 1), - ]) - if (Hash256(block.parentHash) !== parent.hash) { - const changed = [block] - - let current = blockNumber - while (Hash256(block.parentHash) !== parent.hash) { - current-- - ;[block, parent] = await Promise.all([ - this.ethereumClient.getBlock(Hash256(block.parentHash)), - this.getKnownBlock(current - 1), - ]) - changed.push(block) - } - - this.reorgedBlocks = changed.reverse().map((block) => ({ - number: block.number, - hash: Hash256(block.hash), - timestamp: block.timestamp, - })) - - return parent.timestamp - } - - const record: BlockRecord = { - number: block.number, - hash: Hash256(block.hash), - timestamp: block.timestamp, - } - await this.blockRepository.addMany([record]) - return block.timestamp - } - - protected async setSafeHeight(height: number): Promise { - await this.indexerRepository.setSafeHeight(this.id, height) - } - - private async getKnownBlock(blockNumber: number): Promise { - const known = await this.blockRepository.findByNumber(blockNumber) - if (known) { - return known - } - const downloaded = await this.ethereumClient.getBlock(blockNumber) - const record: BlockRecord = { - number: downloaded.number, - hash: Hash256(downloaded.hash), - timestamp: downloaded.timestamp, - } - await this.blockRepository.addMany([record]) - return record - } -} diff --git a/packages/uif/example/ConfigurableIndexer.example.ts b/packages/uif/example/ConfigurableIndexer.example.ts deleted file mode 100644 index 59c351a8..00000000 --- a/packages/uif/example/ConfigurableIndexer.example.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { Logger } from '@l2beat/backend-tools' - -import { ChildIndexer } from '../src/BaseIndexer' -import { HourlyIndexer } from './HourlyIndexer.example' - -interface IndexerDataRepository { - addData(data: number[]): Promise - removeAfter(number: number): Promise -} - -interface IndexerStateRepository { - getSafeHeight(): Promise - setSafeHeight(height: number): Promise - getConfigHash(): Promise - setConfigHash(hash: string): Promise -} -interface Config { - number: number - getConfigHash(): string -} - -export class ConfigurableIndexer extends ChildIndexer { - constructor( - logger: Logger, - parentIndexer: HourlyIndexer, - private readonly config: Config, - private readonly dataRepository: IndexerDataRepository, - private readonly stateRepository: IndexerStateRepository, - ) { - super(logger, [parentIndexer]) - } - - override async initialize(): Promise { - const oldConfigHash = await this.stateRepository.getConfigHash() - const newConfigHash = this.config.getConfigHash() - if (oldConfigHash !== newConfigHash) { - await this.stateRepository.setSafeHeight(0) - await this.stateRepository.setConfigHash(newConfigHash) - } - return this.stateRepository.getSafeHeight() - } - - override async update(from: number, to: number): Promise { - const data: number[] = [] - for (let i = from + 1; i <= to; i++) { - data.push(i) - } - - await this.dataRepository.addData(data) - return to - } - - override async invalidate(targetHeight: number): Promise { - await this.dataRepository.removeAfter(targetHeight) - return targetHeight - } - - override async setSafeHeight(height: number): Promise { - return this.stateRepository.setSafeHeight(height) - } -} diff --git a/packages/uif/example/HourlyIndexer.example.ts b/packages/uif/example/HourlyIndexer.example.ts deleted file mode 100644 index 2197bfe6..00000000 --- a/packages/uif/example/HourlyIndexer.example.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { RootIndexer } from '../src/BaseIndexer' - -export class HourlyIndexer extends RootIndexer { - // initialize is only called once when the indexer is started - async initialize() { - // check every second - setInterval(() => this.requestTick(), 1000) - return this.tick() - } - - // tick is called every time we request a new tick - // It should return the new target height - tick(): Promise { - const ONE_HOUR = 60 * 60 * 1000 - const now = Date.now() - const lastHour = now - (now % ONE_HOUR) - return Promise.resolve(lastHour) - } -} diff --git a/packages/uif/tsconfig.example.json b/packages/uif/tsconfig.example.json deleted file mode 100644 index 7367ec84..00000000 --- a/packages/uif/tsconfig.example.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extends": "./tsconfig.json", - "compilerOptions": { - "noEmit": true - }, - "include": ["example"] -} From 4b7d8bfa8484599f0866c00b218727586b208ec6 Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Thu, 21 Mar 2024 14:09:56 +0100 Subject: [PATCH 05/31] Make indexers safer --- packages/uif/src/BaseIndexer.test.ts | 4 +- packages/uif/src/BaseIndexer.ts | 49 +++-------- packages/uif/src/Indexer.ts | 11 --- packages/uif/src/index.ts | 6 +- packages/uif/src/indexers/ChildIndexer.ts | 85 +++++++++++++++++++ packages/uif/src/indexers/RootIndexer.ts | 57 +++++++++++++ .../src/{ => indexers}/SliceIndexer.test.ts | 8 +- .../uif/src/{ => indexers}/SliceIndexer.ts | 3 +- 8 files changed, 166 insertions(+), 57 deletions(-) delete mode 100644 packages/uif/src/Indexer.ts create mode 100644 packages/uif/src/indexers/ChildIndexer.ts create mode 100644 packages/uif/src/indexers/RootIndexer.ts rename packages/uif/src/{ => indexers}/SliceIndexer.test.ts (97%) rename packages/uif/src/{ => indexers}/SliceIndexer.ts (97%) diff --git a/packages/uif/src/BaseIndexer.test.ts b/packages/uif/src/BaseIndexer.test.ts index 2dbce739..e729ae50 100644 --- a/packages/uif/src/BaseIndexer.test.ts +++ b/packages/uif/src/BaseIndexer.test.ts @@ -2,7 +2,9 @@ import { Logger } from '@l2beat/backend-tools' import { install } from '@sinonjs/fake-timers' import { expect, mockFn } from 'earl' -import { BaseIndexer, ChildIndexer, RootIndexer } from './BaseIndexer' +import { BaseIndexer } from './BaseIndexer' +import { ChildIndexer } from './indexers/ChildIndexer' +import { RootIndexer } from './indexers/RootIndexer' import { IndexerAction } from './reducer/types/IndexerAction' import { RetryStrategy } from './Retries' diff --git a/packages/uif/src/BaseIndexer.ts b/packages/uif/src/BaseIndexer.ts index 1d1e47eb..eb5dbc24 100644 --- a/packages/uif/src/BaseIndexer.ts +++ b/packages/uif/src/BaseIndexer.ts @@ -4,7 +4,6 @@ import { Logger } from '@l2beat/backend-tools' import { assertUnreachable } from './assertUnreachable' import { Height } from './height' -import { Indexer } from './Indexer' import { getInitialState } from './reducer/getInitialState' import { indexerReducer } from './reducer/indexerReducer' import { IndexerAction } from './reducer/types/IndexerAction' @@ -17,8 +16,8 @@ import { import { IndexerState } from './reducer/types/IndexerState' import { Retries, RetryStrategy } from './Retries' -export abstract class BaseIndexer implements Indexer { - private readonly children: Indexer[] = [] +export abstract class BaseIndexer { + private readonly children: BaseIndexer[] = [] /** * This can be overridden to provide a custom retry strategy. It will be @@ -45,7 +44,7 @@ export abstract class BaseIndexer implements Indexer { * Since a root indexer probably doesn't save the height to a database, it * can `return this.tick()` instead. */ - protected abstract initialize(): Promise + abstract initialize(): Promise /** * Saves the height (most likely to a database). The height given is the @@ -55,7 +54,7 @@ export abstract class BaseIndexer implements Indexer { * When `initialize` is called it is expected that it will read the same * height that was saved here. */ - protected abstract setSafeHeight(height: number | null): Promise + abstract setSafeHeight(height: number | null): Promise /** * This method should only be implemented for a child indexer. @@ -79,7 +78,7 @@ export abstract class BaseIndexer implements Indexer { * invalidation down to the returned value. Returning `null` will invalidate * all data. Returning a value greater than `targetHeight` is not permitted. */ - protected abstract update( + abstract update( currentHeight: number | null, targetHeight: number, ): Promise @@ -106,9 +105,7 @@ export abstract class BaseIndexer implements Indexer { * data. Returning a value greater than `targetHeight` means that the indexer * has invalidated down to that height. */ - protected abstract invalidate( - targetHeight: number | null, - ): Promise + abstract invalidate(targetHeight: number | null): Promise /** * This method should only be implemented for a root indexer. @@ -119,7 +116,7 @@ export abstract class BaseIndexer implements Indexer { * As opposed to `update` and `invalidate`, this method cannot return * `null`. */ - protected abstract tick(): Promise + abstract tick(): Promise private state: IndexerState private started = false @@ -129,7 +126,7 @@ export abstract class BaseIndexer implements Indexer { constructor( protected logger: Logger, - public readonly parents: Indexer[], + public readonly parents: BaseIndexer[], opts?: { tickRetryStrategy?: RetryStrategy updateRetryStrategy?: RetryStrategy @@ -164,20 +161,20 @@ export abstract class BaseIndexer implements Indexer { }) } - subscribe(child: Indexer): void { + subscribe(child: BaseIndexer): void { assert(!this.started, 'Indexer already started') this.logger.debug('Child subscribed', { child: child.constructor.name }) this.children.push(child) } - notifyReady(child: Indexer): void { + notifyReady(child: BaseIndexer): void { this.logger.debug('Someone is ready', { child: child.constructor.name }) const index = this.children.indexOf(child) assert(index !== -1, 'Received ready from unknown child') this.dispatch({ type: 'ChildReady', index }) } - notifyUpdate(parent: Indexer, safeHeight: number | null): void { + notifyUpdate(parent: BaseIndexer, safeHeight: number | null): void { this.logger.debug('Someone has updated', { parent: parent.constructor.name, }) @@ -349,27 +346,3 @@ export abstract class BaseIndexer implements Indexer { // #endregion } - -export abstract class RootIndexer extends BaseIndexer { - constructor(logger: Logger, opts?: { tickRetryStrategy?: RetryStrategy }) { - super(logger, [], opts) - } - - override async update(): Promise { - return Promise.reject(new Error('RootIndexer cannot update')) - } - - override async invalidate(): Promise { - return Promise.reject(new Error('RootIndexer cannot invalidate')) - } - - override async setSafeHeight(): Promise { - return Promise.resolve() - } -} - -export abstract class ChildIndexer extends BaseIndexer { - override async tick(): Promise { - return Promise.reject(new Error('ChildIndexer cannot tick')) - } -} diff --git a/packages/uif/src/Indexer.ts b/packages/uif/src/Indexer.ts deleted file mode 100644 index bd5d7a26..00000000 --- a/packages/uif/src/Indexer.ts +++ /dev/null @@ -1,11 +0,0 @@ -export interface UpdateEvent { - type: 'update' - height: number -} - -export interface Indexer { - subscribe(child: Indexer): void - notifyReady(child: Indexer): void - notifyUpdate(parent: Indexer, safeHeight: number | null): void - start(): Promise -} diff --git a/packages/uif/src/index.ts b/packages/uif/src/index.ts index b4968e6d..273a1e04 100644 --- a/packages/uif/src/index.ts +++ b/packages/uif/src/index.ts @@ -1,4 +1,6 @@ export * from './BaseIndexer' -export * from './Indexer' +export * from './height' +export * from './indexers/ChildIndexer' +export * from './indexers/RootIndexer' +export * from './indexers/SliceIndexer' export * from './Retries' -export * from './SliceIndexer' diff --git a/packages/uif/src/indexers/ChildIndexer.ts b/packages/uif/src/indexers/ChildIndexer.ts new file mode 100644 index 00000000..6ce20340 --- /dev/null +++ b/packages/uif/src/indexers/ChildIndexer.ts @@ -0,0 +1,85 @@ +import { BaseIndexer } from '../BaseIndexer' + +/** + * Because of the way TypeScript works, all child indexers need to + * `extends ChildIndexer` and `implements IChildIndexer`. Otherwise it + * is possible to have incorrect method signatures and TypeScript won't + * catch it. + */ +export interface IChildIndexer { + /** + * Initializes the indexer. It should return a height that the indexer has + * synced up to. If the indexer has not synced any data, it should return + * `null`. + * + * This method is expected to read the height that was saved previously with + * `setSafeHeight`. It shouldn't call `setSafeHeight` itself. + */ + initialize: () => Promise + + /** + * Saves the height (most likely to a database). The height given is the + * smallest height from all parents and what the indexer itself synced to + * previously. It can be `null`. + * + * When `initialize` is called it is expected that it will read the same + * height that was saved here. + */ + setSafeHeight: (height: number | null) => Promise + + /** + * Implements the main data fetching process. It is up to the indexer to + * decide how much data to fetch. For example given `.update(100, 200)`, the + * indexer can only fetch data up to 110 and return 110. The next time this + * method will be called with `.update(110, 200)`. + * + * @param currentHeight The height that the indexer has synced up to previously. Can + * be `null` if no data was synced. This value is exclusive so the indexer + * should not fetch data for this height. + * + * @param targetHeight The height that the indexer should sync up to. This value is + * inclusive so the indexer should eventually fetch data for this height. + * + * @returns The height that the indexer has synced up to. Returning + * `currentHeight` means that the indexer has not synced any data. Returning + * a value greater than `currentHeight` means that the indexer has synced up + * to that height. Returning a value less than `currentHeight` will trigger + * invalidation down to the returned value. Returning `null` will invalidate + * all data. Returning a value greater than `targetHeight` is not permitted. + */ + update: ( + currentHeight: number | null, + targetHeight: number, + ) => Promise + + /** + * Responsible for invalidating data that was synced previously. It is + * possible that no data was synced and this method is still called. + * + * Invalidation can, but doesn't have to remove data from the database. If + * you only want to rely on the safe height, you can just return the target + * height and the system will take care of the rest. + * + * This method doesn't have to invalidate all data. If you want to do it in + * steps, you can return a height that is larger than the target height. + * + * @param targetHeight The height that the indexer should invalidate down to. + * Can be `null`. If it is `null`, the indexer should invalidate all + * data. + * + * @returns The height that the indexer has invalidated down to. Returning + * `targetHeight` means that the indexer has invalidated all the required + * data. Returning a value greater than `targetHeight` means that the indexer + * has invalidated down to that height. + */ + invalidate: (targetHeight: number | null) => Promise +} + +export abstract class ChildIndexer + extends BaseIndexer + implements IChildIndexer +{ + override async tick(): Promise { + return Promise.reject(new Error('ChildIndexer cannot tick')) + } +} diff --git a/packages/uif/src/indexers/RootIndexer.ts b/packages/uif/src/indexers/RootIndexer.ts new file mode 100644 index 00000000..2febb4b8 --- /dev/null +++ b/packages/uif/src/indexers/RootIndexer.ts @@ -0,0 +1,57 @@ +import { Logger } from '@l2beat/backend-tools' + +import { BaseIndexer } from '../BaseIndexer' +import { RetryStrategy } from '../Retries' +/** + * Because of the way TypeScript works, all child indexers need to + * `extends RootIndexer` and `implements IRootIndexer`. Otherwise it + * is possible to have incorrect method signatures and TypeScript won't + * catch it. + */ +export interface IRootIndexer { + /** + * Initializes the indexer and returns the initial target height for the + * entire system. If `setSafeHeight` is implemented it should return the + * height that was saved previously. If not it can `return this.tick()`. + * + * This method should also schedule a process to request + * ticks. For example with `setInterval(() => this.requestTick(), 1000)`. + */ + initialize: () => Promise + + /** + * This method is responsible for providing the target height for the entire + * system. Some candidates for this are: the current time or the latest block + * number. + * + * This method cannot return `null`. + */ + tick: () => Promise + + /** + * An optional method for saving the height (most likely to a database). The + * height can be `null`. + * + * When `initialize` is called it is expected that it will read the same + * height that was saved here. + */ + setSafeHeight?: (height: number | null) => Promise +} + +export abstract class RootIndexer extends BaseIndexer implements IRootIndexer { + constructor(logger: Logger, opts?: { tickRetryStrategy?: RetryStrategy }) { + super(logger, [], opts) + } + + override async update(): Promise { + return Promise.reject(new Error('RootIndexer cannot update')) + } + + override async invalidate(): Promise { + return Promise.reject(new Error('RootIndexer cannot invalidate')) + } + + override async setSafeHeight(): Promise { + return Promise.resolve() + } +} diff --git a/packages/uif/src/SliceIndexer.test.ts b/packages/uif/src/indexers/SliceIndexer.test.ts similarity index 97% rename from packages/uif/src/SliceIndexer.test.ts rename to packages/uif/src/indexers/SliceIndexer.test.ts index 5273e6e3..d2d9d213 100644 --- a/packages/uif/src/SliceIndexer.test.ts +++ b/packages/uif/src/indexers/SliceIndexer.test.ts @@ -1,10 +1,10 @@ import { Logger } from '@l2beat/backend-tools' import { expect, mockFn } from 'earl' -import { BaseIndexer } from './BaseIndexer' -import { TestRootIndexer, waitUntil } from './BaseIndexer.test' -import { IndexerAction } from './reducer/types/IndexerAction' -import { RetryStrategy } from './Retries' +import { BaseIndexer } from '../BaseIndexer' +import { TestRootIndexer, waitUntil } from '../BaseIndexer.test' +import { IndexerAction } from '../reducer/types/IndexerAction' +import { RetryStrategy } from '../Retries' import { diffSlices, SliceHash, diff --git a/packages/uif/src/SliceIndexer.ts b/packages/uif/src/indexers/SliceIndexer.ts similarity index 97% rename from packages/uif/src/SliceIndexer.ts rename to packages/uif/src/indexers/SliceIndexer.ts index 88f06f56..7ecbc916 100644 --- a/packages/uif/src/SliceIndexer.ts +++ b/packages/uif/src/indexers/SliceIndexer.ts @@ -1,4 +1,4 @@ -import { ChildIndexer } from './BaseIndexer' +import { ChildIndexer } from './ChildIndexer' export type SliceHash = string @@ -13,6 +13,7 @@ export interface SliceUpdate { to: number } +// TODO: implements IChildIndexer export abstract class SliceIndexer extends ChildIndexer { override async update(from: number, to: number): Promise { const sliceStates = await this.getSliceStates() From 29811b3a60d2b557520505dd0ee7651fa916f782 Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Thu, 21 Mar 2024 15:41:10 +0100 Subject: [PATCH 06/31] Add stub MultiIndexer --- packages/uif/src/BaseIndexer.ts | 4 + packages/uif/src/index.ts | 3 +- .../uif/src/indexers/multi/MultiIndexer.ts | 93 +++++++++ .../indexers/multi/diffConfigurations.test.ts | 195 ++++++++++++++++++ .../src/indexers/multi/diffConfigurations.ts | 83 ++++++++ packages/uif/src/indexers/multi/types.ts | 26 +++ 6 files changed, 403 insertions(+), 1 deletion(-) create mode 100644 packages/uif/src/indexers/multi/MultiIndexer.ts create mode 100644 packages/uif/src/indexers/multi/diffConfigurations.test.ts create mode 100644 packages/uif/src/indexers/multi/diffConfigurations.ts create mode 100644 packages/uif/src/indexers/multi/types.ts diff --git a/packages/uif/src/BaseIndexer.ts b/packages/uif/src/BaseIndexer.ts index eb5dbc24..96a960d3 100644 --- a/packages/uif/src/BaseIndexer.ts +++ b/packages/uif/src/BaseIndexer.ts @@ -150,6 +150,10 @@ export abstract class BaseIndexer { opts?.invalidateRetryStrategy ?? BaseIndexer.GET_DEFAULT_RETRY_STRATEGY() } + get safeHeight(): number | null { + return this.state.safeHeight + } + async start(): Promise { assert(!this.started, 'Indexer already started') this.started = true diff --git a/packages/uif/src/index.ts b/packages/uif/src/index.ts index 273a1e04..181838a4 100644 --- a/packages/uif/src/index.ts +++ b/packages/uif/src/index.ts @@ -1,6 +1,7 @@ export * from './BaseIndexer' export * from './height' export * from './indexers/ChildIndexer' +export * from './indexers/multi/MultiIndexer' +export type { Configuration } from './indexers/multi/types' export * from './indexers/RootIndexer' -export * from './indexers/SliceIndexer' export * from './Retries' diff --git a/packages/uif/src/indexers/multi/MultiIndexer.ts b/packages/uif/src/indexers/multi/MultiIndexer.ts new file mode 100644 index 00000000..76fd339b --- /dev/null +++ b/packages/uif/src/indexers/multi/MultiIndexer.ts @@ -0,0 +1,93 @@ +import { Logger } from '@l2beat/backend-tools' + +import { BaseIndexer } from '../../BaseIndexer' +import { RetryStrategy } from '../../Retries' +import { ChildIndexer, IChildIndexer } from '../ChildIndexer' +import { diffConfigurations } from './diffConfigurations' +import { + Configuration, + ConfigurationRange, + RemovalConfiguration, + StoredConfiguration, +} from './types' + +export interface IMultiIndexer { + multiInitialize: () => Promise[]> + removeData: (configurations: RemovalConfiguration[]) => Promise + setStoredConfigurations: ( + configurations: StoredConfiguration[], + ) => Promise +} + +export abstract class MultiIndexer + extends ChildIndexer + implements IChildIndexer, IMultiIndexer +{ + constructor( + logger: Logger, + parents: BaseIndexer[], + readonly configurations: Configuration[], + opts?: { + updateRetryStrategy?: RetryStrategy + invalidateRetryStrategy?: RetryStrategy + }, + ) { + super(logger, parents, opts) + } + + abstract multiInitialize(): Promise[]> + abstract removeData(configurations: RemovalConfiguration[]): Promise + abstract setStoredConfigurations( + configurations: StoredConfiguration[], + ): Promise + + async initialize(): Promise { + const stored = await this.multiInitialize() + const { toRemove, safeHeight } = diffConfigurations( + this.configurations, + stored, + ) + await this.removeData(toRemove) + return safeHeight + } + + async update(from: number | null, to: number): Promise { + return Promise.resolve(to) + } + + async invalidate(targetHeight: number | null): Promise { + return Promise.resolve(targetHeight) + } + + async setSafeHeight(): Promise { + return Promise.resolve() + } +} + +export function toRanges( + configurations: Configuration[], +): ConfigurationRange[] { + let currentRange: ConfigurationRange = { + from: null, + to: null, + configurations: [], + } + const ranges: ConfigurationRange[] = [currentRange] + + const sorted = [...configurations].sort((a, b) => a.minHeight - b.minHeight) + for (const configuration of sorted) { + if (configuration.minHeight === currentRange.from) { + currentRange.configurations.push(configuration) + } else { + currentRange.to = configuration.minHeight - 1 + currentRange = { + from: configuration.minHeight, + to: null, + configurations: [configuration], + } + ranges.push(currentRange) + } + } + + return ranges +} diff --git a/packages/uif/src/indexers/multi/diffConfigurations.test.ts b/packages/uif/src/indexers/multi/diffConfigurations.test.ts new file mode 100644 index 00000000..4d401771 --- /dev/null +++ b/packages/uif/src/indexers/multi/diffConfigurations.test.ts @@ -0,0 +1,195 @@ +import { expect } from 'earl' + +import { diffConfigurations } from './diffConfigurations' + +describe(diffConfigurations.name, () => { + describe('errors', () => { + it('duplicate config id', () => { + expect(() => + diffConfigurations([actual('a', 100, null), actual('a', 200, 300)], []), + ).toThrow(/a is duplicated/) + }) + + it('minHeight greater than maxHeight', () => { + expect(() => diffConfigurations([actual('a', 200, 100)], [])).toThrow( + /a has minHeight greater than maxHeight/, + ) + }) + }) + + describe('regular sync', () => { + it('empty actual and stored', () => { + const result = diffConfigurations([], []) + expect(result).toEqual({ toRemove: [], safeHeight: Infinity }) + }) + + it('empty stored', () => { + const result = diffConfigurations( + [actual('a', 100, null), actual('b', 200, 300)], + [], + ) + expect(result).toEqual({ toRemove: [], safeHeight: 99 }) + }) + + it('partially synced, both early', () => { + const result = diffConfigurations( + [actual('a', 100, 400), actual('b', 200, null)], + [stored('a', 100, 300), stored('b', 200, 300)], + ) + expect(result).toEqual({ + toRemove: [], + safeHeight: 300, + }) + }) + + it('partially synced, one not yet started', () => { + const result = diffConfigurations( + [actual('a', 100, 400), actual('b', 555, null)], + [stored('a', 100, 300)], + ) + expect(result).toEqual({ + toRemove: [], + safeHeight: 300, + }) + }) + + it('partially synced, one finished', () => { + const result = diffConfigurations( + [actual('a', 100, 555), actual('b', 200, 300)], + [stored('a', 100, 400), stored('b', 200, 300)], + ) + expect(result).toEqual({ + toRemove: [], + safeHeight: 400, + }) + }) + + it('partially synced, one finished, one infinite', () => { + const result = diffConfigurations( + [actual('a', 100, null), actual('b', 200, 300)], + [stored('a', 100, 400), stored('b', 200, 300)], + ) + expect(result).toEqual({ + toRemove: [], + safeHeight: 400, + }) + }) + + it('both synced', () => { + const result = diffConfigurations( + [actual('a', 100, 400), actual('b', 200, 300)], + [stored('a', 100, 400), stored('b', 200, 300)], + ) + expect(result).toEqual({ + toRemove: [], + safeHeight: Infinity, + }) + }) + }) + + describe('configuration changed', () => { + it('empty actual', () => { + const result = diffConfigurations( + [], + [stored('a', 100, 300), stored('b', 200, 300)], + ) + expect(result).toEqual({ + toRemove: [removal('a', 100, 300), removal('b', 200, 300)], + safeHeight: Infinity, + }) + }) + + it('single removed', () => { + const result = diffConfigurations( + [actual('b', 200, 400)], + [stored('a', 100, 300), stored('b', 200, 300)], + ) + expect(result).toEqual({ + toRemove: [removal('a', 100, 300)], + safeHeight: 300, + }) + }) + + it('single removed', () => { + const result = diffConfigurations( + [actual('b', 200, 400)], + [stored('a', 100, 300), stored('b', 200, 300)], + ) + expect(result).toEqual({ + toRemove: [removal('a', 100, 300)], + safeHeight: 300, + }) + }) + + it('maxHeight updated up', () => { + const result = diffConfigurations( + [actual('a', 100, 400)], + [stored('a', 100, 300)], + ) + expect(result).toEqual({ + toRemove: [], + safeHeight: 300, + }) + }) + + it('maxHeight updated down', () => { + const result = diffConfigurations( + [actual('a', 100, 200)], + [stored('a', 100, 300)], + ) + expect(result).toEqual({ + toRemove: [removal('a', 201, 300)], + safeHeight: Infinity, + }) + }) + + it('minHeight updated up', () => { + const result = diffConfigurations( + [actual('a', 200, 400)], + [stored('a', 100, 300)], + ) + expect(result).toEqual({ + toRemove: [removal('a', 100, 199)], + safeHeight: 300, + }) + }) + + it('minHeight updated down', () => { + const result = diffConfigurations( + [actual('a', 100, 400)], + [stored('a', 200, 300)], + ) + expect(result).toEqual({ + toRemove: [removal('a', 200, 300)], + safeHeight: 99, + }) + }) + + it('both min and max height updated', () => { + const result = diffConfigurations( + [actual('a', 200, 300)], + [stored('a', 100, 400)], + ) + expect(result).toEqual({ + toRemove: [removal('a', 100, 199), removal('a', 301, 400)], + safeHeight: Infinity, + }) + }) + }) +}) + +function actual(id: string, minHeight: number, maxHeight: number | null) { + return { id, properties: null, minHeight, maxHeight } +} + +function stored(id: string, minHeight: number, currentHeight: number) { + return { id, properties: null, minHeight, currentHeight } +} + +function removal( + id: string, + fromHeightInclusive: number, + toHeightInclusive: number, +) { + return { id, properties: null, fromHeightInclusive, toHeightInclusive } +} diff --git a/packages/uif/src/indexers/multi/diffConfigurations.ts b/packages/uif/src/indexers/multi/diffConfigurations.ts new file mode 100644 index 00000000..1286c053 --- /dev/null +++ b/packages/uif/src/indexers/multi/diffConfigurations.ts @@ -0,0 +1,83 @@ +import { Height } from '../../height' +import { + Configuration, + RemovalConfiguration, + StoredConfiguration, +} from './types' + +export function diffConfigurations( + actual: Configuration[], + stored: StoredConfiguration[], +): { + toRemove: RemovalConfiguration[] + safeHeight: number | null +} { + let safeHeight: number | null = Infinity + + const knownIds = new Set() + const actualMap = new Map(actual.map((c) => [c.id, c])) + const storedMap = new Map(stored.map((c) => [c.id, c])) + + const toRemove: RemovalConfiguration[] = stored + .filter((c) => !actualMap.has(c.id)) + .map((c) => ({ + id: c.id, + properties: c.properties, + fromHeightInclusive: c.minHeight, + toHeightInclusive: c.currentHeight, + })) + + for (const c of actual) { + if (knownIds.has(c.id)) { + throw new Error(`Configuration ${c.id} is duplicated!`) + } + knownIds.add(c.id) + + if (c.maxHeight !== null && c.minHeight > c.maxHeight) { + throw new Error( + `Configuration ${c.id} has minHeight greater than maxHeight!`, + ) + } + + const stored = storedMap.get(c.id) + if (!stored) { + safeHeight = Height.min(safeHeight, c.minHeight - 1) + continue + } + + if (stored.minHeight > c.minHeight) { + safeHeight = Height.min(safeHeight, c.minHeight - 1) + // We remove everything because we cannot have gaps in downloaded data + // We will re-download everything from the beginning + toRemove.push({ + id: stored.id, + properties: stored.properties, + fromHeightInclusive: stored.minHeight, + toHeightInclusive: stored.currentHeight, + }) + } else if (stored.minHeight < c.minHeight) { + toRemove.push({ + id: stored.id, + properties: stored.properties, + fromHeightInclusive: stored.minHeight, + toHeightInclusive: c.minHeight - 1, + }) + } + + if (c.maxHeight !== null && stored.currentHeight > c.maxHeight) { + toRemove.push({ + id: stored.id, + properties: stored.properties, + fromHeightInclusive: c.maxHeight + 1, + toHeightInclusive: stored.currentHeight, + }) + } else if ( + c.maxHeight === null || + Height.lt(stored.currentHeight, c.maxHeight) + ) { + safeHeight = Height.min(safeHeight, stored.currentHeight) + } + } + + return { toRemove, safeHeight } +} diff --git a/packages/uif/src/indexers/multi/types.ts b/packages/uif/src/indexers/multi/types.ts new file mode 100644 index 00000000..ce84202b --- /dev/null +++ b/packages/uif/src/indexers/multi/types.ts @@ -0,0 +1,26 @@ +export interface Configuration { + id: string + properties: T + minHeight: number + maxHeight: number | null +} + +export interface StoredConfiguration { + id: string + properties: T + minHeight: number + currentHeight: number +} + +export interface RemovalConfiguration { + id: string + properties: T + fromHeightInclusive: number + toHeightInclusive: number +} + +export interface ConfigurationRange { + from: number | null + to: number | null + configurations: Configuration[] +} From b14619f3f1aadae09f0f9fc3ac6e1974e6005be0 Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Thu, 21 Mar 2024 16:46:25 +0100 Subject: [PATCH 07/31] Implement toRanges --- .../uif/src/indexers/multi/MultiIndexer.ts | 111 +++++++++++++----- .../uif/src/indexers/multi/toRanges.test.ts | 83 +++++++++++++ packages/uif/src/indexers/multi/toRanges.ts | 44 +++++++ packages/uif/src/indexers/multi/types.ts | 4 +- 4 files changed, 210 insertions(+), 32 deletions(-) create mode 100644 packages/uif/src/indexers/multi/toRanges.test.ts create mode 100644 packages/uif/src/indexers/multi/toRanges.ts diff --git a/packages/uif/src/indexers/multi/MultiIndexer.ts b/packages/uif/src/indexers/multi/MultiIndexer.ts index 76fd339b..2e65dad9 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.ts @@ -1,9 +1,11 @@ import { Logger } from '@l2beat/backend-tools' import { BaseIndexer } from '../../BaseIndexer' +import { Height } from '../../height' import { RetryStrategy } from '../../Retries' import { ChildIndexer, IChildIndexer } from '../ChildIndexer' import { diffConfigurations } from './diffConfigurations' +import { toRanges } from './toRanges' import { Configuration, ConfigurationRange, @@ -12,8 +14,52 @@ import { } from './types' export interface IMultiIndexer { + /** + * Initializes the indexer. It returns the configurations that were saved + * previously. In case no configurations were saved, it should return an empty + * array. + * + * This method is expected to read the configurations that was saved + * previously with `setStoredConfigurations`. It shouldn't call + * `setStoredConfigurations` itself. + */ multiInitialize: () => Promise[]> + + /** + * Removes data that was previously synced but because configurations changed + * is no longer valid. The data should be removed for the ranges specified + * in each configuration. It is possible for multiple ranges to share a + * configuration id! + * + * This method is only called during the initialization of the indexer, after + * `multiInitialize` returns. + */ removeData: (configurations: RemovalConfiguration[]) => Promise + + /** + * + * @param currentHeight The height that the indexer has synced up to previously. Can + * be `null` if no data was synced. This value is exclusive so the indexer + * should not fetch data for this height. + * + * @param targetHeight The height that the indexer should sync up to. This value is + * inclusive so the indexer should eventually fetch data for this height. + * + * @param configurations foo + * + * @returns The height that the indexer has synced up to. Returning + * `currentHeight` means that the indexer has not synced any data. Returning + * a value greater than `currentHeight` means that the indexer has synced up + * to that height. Returning a value less than `currentHeight` will trigger + * invalidation down to the returned value. Returning `null` will invalidate + * all data. Returning a value greater than `targetHeight` is not permitted. + */ + multiUpdate: ( + currentHeight: number | null, + targetHeight: number, + configurations: Configuration[], + ) => Promise + setStoredConfigurations: ( configurations: StoredConfiguration[], ) => Promise @@ -23,6 +69,8 @@ export abstract class MultiIndexer extends ChildIndexer implements IChildIndexer, IMultiIndexer { + private readonly ranges: ConfigurationRange[] + constructor( logger: Logger, parents: BaseIndexer[], @@ -33,9 +81,15 @@ export abstract class MultiIndexer }, ) { super(logger, parents, opts) + this.ranges = toRanges(configurations) } abstract multiInitialize(): Promise[]> + abstract multiUpdate( + currentHeight: number | null, + targetHeight: number, + configurations: Configuration[], + ): Promise abstract removeData(configurations: RemovalConfiguration[]): Promise abstract setStoredConfigurations( configurations: StoredConfiguration[], @@ -51,8 +105,33 @@ export abstract class MultiIndexer return safeHeight } - async update(from: number | null, to: number): Promise { - return Promise.resolve(to) + async update( + currentHeight: number | null, + targetHeight: number, + ): Promise { + const range = this.ranges.find((range) => + Height.gt(range.from, currentHeight), + ) + if (!range) { + throw new Error('Programmer error, there should always be a range') + } + if (range.configurations.length === 0) { + return Height.min(range.to, targetHeight) + } + + const minTarget = Math.min(range.to, targetHeight) + + const newHeight = await this.multiUpdate( + currentHeight, + minTarget, + range.configurations, + ) + if (Height.gt(newHeight, minTarget)) { + throw new Error( + 'Programmer error, returned height cannot be greater than targetHeight', + ) + } + return newHeight } async invalidate(targetHeight: number | null): Promise { @@ -63,31 +142,3 @@ export abstract class MultiIndexer return Promise.resolve() } } - -export function toRanges( - configurations: Configuration[], -): ConfigurationRange[] { - let currentRange: ConfigurationRange = { - from: null, - to: null, - configurations: [], - } - const ranges: ConfigurationRange[] = [currentRange] - - const sorted = [...configurations].sort((a, b) => a.minHeight - b.minHeight) - for (const configuration of sorted) { - if (configuration.minHeight === currentRange.from) { - currentRange.configurations.push(configuration) - } else { - currentRange.to = configuration.minHeight - 1 - currentRange = { - from: configuration.minHeight, - to: null, - configurations: [configuration], - } - ranges.push(currentRange) - } - } - - return ranges -} diff --git a/packages/uif/src/indexers/multi/toRanges.test.ts b/packages/uif/src/indexers/multi/toRanges.test.ts new file mode 100644 index 00000000..432bce04 --- /dev/null +++ b/packages/uif/src/indexers/multi/toRanges.test.ts @@ -0,0 +1,83 @@ +import { expect } from 'earl' + +import { toRanges } from './toRanges' + +describe(toRanges.name, () => { + it('empty', () => { + const ranges = toRanges([]) + expect(ranges).toEqual([ + { from: -Infinity, to: Infinity, configurations: [] }, + ]) + }) + + it('single infinite configuration', () => { + const ranges = toRanges([config('a', 100, null)]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { from: 100, to: Infinity, configurations: [config('a', 100, null)] }, + ]) + }) + + it('single finite configuration', () => { + const ranges = toRanges([config('a', 100, 300)]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { from: 100, to: 300, configurations: [config('a', 100, 300)] }, + { from: 301, to: Infinity, configurations: [] }, + ]) + }) + + it('multiple overlapping configurations', () => { + const ranges = toRanges([ + config('a', 100, 300), + config('b', 200, 400), + config('c', 300, 500), + ]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { from: 100, to: 199, configurations: [config('a', 100, 300)] }, + { + from: 200, + to: 299, + configurations: [config('a', 100, 300), config('b', 200, 400)], + }, + { + from: 300, + to: 300, + configurations: [ + config('a', 100, 300), + config('b', 200, 400), + config('c', 300, 500), + ], + }, + { + from: 301, + to: 400, + configurations: [config('b', 200, 400), config('c', 300, 500)], + }, + { from: 401, to: 500, configurations: [config('c', 300, 500)] }, + { from: 501, to: Infinity, configurations: [] }, + ]) + }) + + it('multiple non-overlapping configurations', () => { + const ranges = toRanges([ + config('a', 100, 200), + config('b', 300, 400), + config('c', 500, 600), + ]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { from: 100, to: 200, configurations: [config('a', 100, 200)] }, + { from: 201, to: 299, configurations: [] }, + { from: 300, to: 400, configurations: [config('b', 300, 400)] }, + { from: 401, to: 499, configurations: [] }, + { from: 500, to: 600, configurations: [config('c', 500, 600)] }, + { from: 601, to: Infinity, configurations: [] }, + ]) + }) +}) + +function config(id: string, minHeight: number, maxHeight: number | null) { + return { id, properties: null, minHeight, maxHeight } +} diff --git a/packages/uif/src/indexers/multi/toRanges.ts b/packages/uif/src/indexers/multi/toRanges.ts new file mode 100644 index 00000000..a99e94b2 --- /dev/null +++ b/packages/uif/src/indexers/multi/toRanges.ts @@ -0,0 +1,44 @@ +import { Configuration, ConfigurationRange } from './types' + +export function toRanges( + configurations: Configuration[], +): ConfigurationRange[] { + const minHeights = configurations.map((c) => c.minHeight) + const maxHeights = configurations + .map((c) => c.maxHeight) + .filter((height): height is number => height !== null) + + const starts = minHeights + .concat(maxHeights.map((height) => height + 1)) + .sort((a, b) => a - b) + .filter((height, i, arr) => arr.indexOf(height) === i) + + let lastRange: ConfigurationRange = { + from: -Infinity, + to: Infinity, + configurations: [], + } + const ranges: ConfigurationRange[] = [lastRange] + for (const start of starts) { + lastRange.to = start - 1 + lastRange = { + from: start, + to: Infinity, + configurations: [], + } + ranges.push(lastRange) + } + + for (const configuration of configurations) { + const min = configuration.minHeight + const max = configuration.maxHeight ?? Infinity + + for (const range of ranges) { + if (!(max < range.from || min > range.to)) { + range.configurations.push(configuration) + } + } + } + + return ranges +} diff --git a/packages/uif/src/indexers/multi/types.ts b/packages/uif/src/indexers/multi/types.ts index ce84202b..e9b3072f 100644 --- a/packages/uif/src/indexers/multi/types.ts +++ b/packages/uif/src/indexers/multi/types.ts @@ -20,7 +20,7 @@ export interface RemovalConfiguration { } export interface ConfigurationRange { - from: number | null - to: number | null + from: number + to: number configurations: Configuration[] } From 7cf906babae0ff63a0ee11c45bba1f8a3e468c7d Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Thu, 21 Mar 2024 17:25:26 +0100 Subject: [PATCH 08/31] Add tests for MultiIndexer --- packages/uif/src/BaseIndexer.test.ts | 16 +-- .../src/indexers/multi/MultiIndexer.test.ts | 105 ++++++++++++++++++ .../uif/src/indexers/multi/MultiIndexer.ts | 12 +- 3 files changed, 122 insertions(+), 11 deletions(-) create mode 100644 packages/uif/src/indexers/multi/MultiIndexer.test.ts diff --git a/packages/uif/src/BaseIndexer.test.ts b/packages/uif/src/BaseIndexer.test.ts index e729ae50..2199f7a6 100644 --- a/packages/uif/src/BaseIndexer.test.ts +++ b/packages/uif/src/BaseIndexer.test.ts @@ -235,7 +235,7 @@ export class TestRootIndexer extends RootIndexer { ticking = false constructor( - private safeHeight: number, + private testSafeHeight: number, name?: string, retryStrategy?: { tickRetryStrategy?: RetryStrategy }, ) { @@ -250,7 +250,7 @@ export class TestRootIndexer extends RootIndexer { async doTick(height: number): Promise { await waitUntil(() => this.getState().status === 'idle') - this.safeHeight = height + this.testSafeHeight = height const counter = this.dispatchCounter this.requestTick() await waitUntil(() => this.dispatchCounter > counter) @@ -267,7 +267,7 @@ export class TestRootIndexer extends RootIndexer { await waitUntil(() => this.dispatchCounter > counter) } - override tick(): Promise { + override async tick(): Promise { this.ticking = true return new Promise((resolve, reject) => { @@ -280,9 +280,9 @@ export class TestRootIndexer extends RootIndexer { override async initialize(): Promise { const promise = this.tick() - this.resolveTick(this.safeHeight) + this.resolveTick(this.testSafeHeight) await promise - return this.safeHeight + return this.testSafeHeight } } @@ -325,7 +325,7 @@ class TestChildIndexer extends ChildIndexer { constructor( parents: BaseIndexer[], - private safeHeight: number, + private testSafeHeight: number, name?: string, retryStrategy?: { invalidateRetryStrategy?: RetryStrategy @@ -342,11 +342,11 @@ class TestChildIndexer extends ChildIndexer { } override initialize(): Promise { - return Promise.resolve(this.safeHeight) + return Promise.resolve(this.testSafeHeight) } override setSafeHeight(safeHeight: number): Promise { - this.safeHeight = safeHeight + this.testSafeHeight = safeHeight return Promise.resolve() } diff --git a/packages/uif/src/indexers/multi/MultiIndexer.test.ts b/packages/uif/src/indexers/multi/MultiIndexer.test.ts new file mode 100644 index 00000000..b8626003 --- /dev/null +++ b/packages/uif/src/indexers/multi/MultiIndexer.test.ts @@ -0,0 +1,105 @@ +import { Logger } from '@l2beat/backend-tools' +import { expect, mockFn } from 'earl' + +import { IMultiIndexer, MultiIndexer } from './MultiIndexer' +import { Configuration, StoredConfiguration } from './types' + +describe(MultiIndexer.name, () => { + describe(MultiIndexer.prototype.update.name, () => { + it('calls multiUpdate with an early matching configuration', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 200), actual('b', 300, 400)], + [], + ) + + const newHeight = await testIndexer.update(100, 500) + + expect(newHeight).toEqual(200) + expect(testIndexer.multiUpdate).toHaveBeenOnlyCalledWith(100, 200, [ + actual('a', 100, 200), + ]) + }) + + it('calls multiUpdate with a late matching configuration', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 200), actual('b', 300, 400)], + [], + ) + + const newHeight = await testIndexer.update(300, 500) + + expect(newHeight).toEqual(400) + expect(testIndexer.multiUpdate).toHaveBeenOnlyCalledWith(300, 400, [ + actual('b', 300, 400), + ]) + }) + + it('calls multiUpdate with a two matching configurations', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 200), actual('b', 100, 400)], + [], + ) + + const newHeight = await testIndexer.update(100, 500) + + expect(newHeight).toEqual(200) + expect(testIndexer.multiUpdate).toHaveBeenOnlyCalledWith(100, 200, [ + actual('a', 100, 200), + actual('b', 100, 400), + ]) + }) + + it('skips calling multiUpdate if we are too early', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 200), actual('b', 300, 400)], + [], + ) + + const newHeight = await testIndexer.update(null, 500) + + expect(newHeight).toEqual(99) + expect(testIndexer.multiUpdate).not.toHaveBeenCalled() + }) + + it('skips calling multiUpdate if we are too late', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 200), actual('b', 300, 400)], + [], + ) + + const newHeight = await testIndexer.update(400, 500) + + expect(newHeight).toEqual(500) + expect(testIndexer.multiUpdate).not.toHaveBeenCalled() + }) + }) +}) + +class TestMultiIndexer + extends MultiIndexer + implements IMultiIndexer +{ + constructor( + configurations: Configuration[], + private readonly _stored: StoredConfiguration[], + ) { + super(Logger.SILENT, [], configurations) + } + + override multiInitialize(): Promise[]> { + return Promise.resolve(this._stored) + } + + multiUpdate = mockFn['multiUpdate']>((_, targetHeight) => + Promise.resolve(targetHeight), + ) + + removeData = mockFn['removeData']>() + + setStoredConfigurations = + mockFn['setStoredConfigurations']>() +} + +function actual(id: string, minHeight: number, maxHeight: number | null) { + return { id, properties: null, minHeight, maxHeight } +} diff --git a/packages/uif/src/indexers/multi/MultiIndexer.ts b/packages/uif/src/indexers/multi/MultiIndexer.ts index 2e65dad9..e9e548eb 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.ts @@ -37,6 +37,10 @@ export interface IMultiIndexer { removeData: (configurations: RemovalConfiguration[]) => Promise /** + * Implements the main data fetching process. It is up to the indexer to + * decide how much data to fetch. For example given `.update(100, 200, [...])`, the + * indexer can only fetch data up to 110 and return 110. The next time this + * method will be called with `.update(110, 200, [...])`. * * @param currentHeight The height that the indexer has synced up to previously. Can * be `null` if no data was synced. This value is exclusive so the indexer @@ -45,7 +49,9 @@ export interface IMultiIndexer { * @param targetHeight The height that the indexer should sync up to. This value is * inclusive so the indexer should eventually fetch data for this height. * - * @param configurations foo + * @param configurations The configurations that the indexer should use to + * sync data. The configurations are guaranteed to be in the range of + * `currentHeight` and `targetHeight`. * * @returns The height that the indexer has synced up to. Returning * `currentHeight` means that the indexer has not synced any data. Returning @@ -109,8 +115,8 @@ export abstract class MultiIndexer currentHeight: number | null, targetHeight: number, ): Promise { - const range = this.ranges.find((range) => - Height.gt(range.from, currentHeight), + const range = this.ranges.find( + (range) => currentHeight === null || range.from >= currentHeight, ) if (!range) { throw new Error('Programmer error, there should always be a range') From 43bab2adf3cb95e2fde2e99999763c5001b79c9f Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Thu, 21 Mar 2024 17:36:21 +0100 Subject: [PATCH 09/31] Fix some edge cases --- .../src/indexers/multi/MultiIndexer.test.ts | 27 +++++++++++++++++++ .../uif/src/indexers/multi/MultiIndexer.ts | 4 ++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/packages/uif/src/indexers/multi/MultiIndexer.test.ts b/packages/uif/src/indexers/multi/MultiIndexer.test.ts index b8626003..3ae0a126 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.test.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.test.ts @@ -49,6 +49,21 @@ describe(MultiIndexer.name, () => { ]) }) + it('calls multiUpdate with a two middle matching configurations', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 400), actual('b', 200, 500)], + [], + ) + + const newHeight = await testIndexer.update(300, 600) + + expect(newHeight).toEqual(400) + expect(testIndexer.multiUpdate).toHaveBeenOnlyCalledWith(300, 400, [ + actual('a', 100, 400), + actual('b', 200, 500), + ]) + }) + it('skips calling multiUpdate if we are too early', async () => { const testIndexer = new TestMultiIndexer( [actual('a', 100, 200), actual('b', 300, 400)], @@ -72,6 +87,18 @@ describe(MultiIndexer.name, () => { expect(newHeight).toEqual(500) expect(testIndexer.multiUpdate).not.toHaveBeenCalled() }) + + it('skips calling multiUpdate between configs', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 200), actual('b', 300, 400)], + [], + ) + + const newHeight = await testIndexer.update(200, 500) + + expect(newHeight).toEqual(299) + expect(testIndexer.multiUpdate).not.toHaveBeenCalled() + }) }) }) diff --git a/packages/uif/src/indexers/multi/MultiIndexer.ts b/packages/uif/src/indexers/multi/MultiIndexer.ts index e9e548eb..e1731eee 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.ts @@ -116,7 +116,9 @@ export abstract class MultiIndexer targetHeight: number, ): Promise { const range = this.ranges.find( - (range) => currentHeight === null || range.from >= currentHeight, + (range) => + currentHeight === null || + (range.from <= currentHeight + 1 && range.to > currentHeight), ) if (!range) { throw new Error('Programmer error, there should always be a range') From 4de54e73ada023f3f623856edd26106e2b2d17eb Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Thu, 21 Mar 2024 19:05:40 +0100 Subject: [PATCH 10/31] Simplify configuration types --- .../uif/src/indexers/multi/MultiIndexer.test.ts | 4 ++-- packages/uif/src/indexers/multi/MultiIndexer.ts | 12 ++++++------ .../src/indexers/multi/diffConfigurations.test.ts | 4 ++-- .../uif/src/indexers/multi/diffConfigurations.ts | 14 +++++--------- packages/uif/src/indexers/multi/types.ts | 6 ++---- 5 files changed, 17 insertions(+), 23 deletions(-) diff --git a/packages/uif/src/indexers/multi/MultiIndexer.test.ts b/packages/uif/src/indexers/multi/MultiIndexer.test.ts index 3ae0a126..9196c3d3 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.test.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.test.ts @@ -108,12 +108,12 @@ class TestMultiIndexer { constructor( configurations: Configuration[], - private readonly _stored: StoredConfiguration[], + private readonly _stored: StoredConfiguration[], ) { super(Logger.SILENT, [], configurations) } - override multiInitialize(): Promise[]> { + override multiInitialize(): Promise { return Promise.resolve(this._stored) } diff --git a/packages/uif/src/indexers/multi/MultiIndexer.ts b/packages/uif/src/indexers/multi/MultiIndexer.ts index e1731eee..e4babe43 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.ts @@ -23,7 +23,7 @@ export interface IMultiIndexer { * previously with `setStoredConfigurations`. It shouldn't call * `setStoredConfigurations` itself. */ - multiInitialize: () => Promise[]> + multiInitialize: () => Promise /** * Removes data that was previously synced but because configurations changed @@ -34,7 +34,7 @@ export interface IMultiIndexer { * This method is only called during the initialization of the indexer, after * `multiInitialize` returns. */ - removeData: (configurations: RemovalConfiguration[]) => Promise + removeData: (configurations: RemovalConfiguration[]) => Promise /** * Implements the main data fetching process. It is up to the indexer to @@ -67,7 +67,7 @@ export interface IMultiIndexer { ) => Promise setStoredConfigurations: ( - configurations: StoredConfiguration[], + configurations: StoredConfiguration[], ) => Promise } @@ -90,15 +90,15 @@ export abstract class MultiIndexer this.ranges = toRanges(configurations) } - abstract multiInitialize(): Promise[]> + abstract multiInitialize(): Promise abstract multiUpdate( currentHeight: number | null, targetHeight: number, configurations: Configuration[], ): Promise - abstract removeData(configurations: RemovalConfiguration[]): Promise + abstract removeData(configurations: RemovalConfiguration[]): Promise abstract setStoredConfigurations( - configurations: StoredConfiguration[], + configurations: StoredConfiguration[], ): Promise async initialize(): Promise { diff --git a/packages/uif/src/indexers/multi/diffConfigurations.test.ts b/packages/uif/src/indexers/multi/diffConfigurations.test.ts index 4d401771..dc38f508 100644 --- a/packages/uif/src/indexers/multi/diffConfigurations.test.ts +++ b/packages/uif/src/indexers/multi/diffConfigurations.test.ts @@ -183,7 +183,7 @@ function actual(id: string, minHeight: number, maxHeight: number | null) { } function stored(id: string, minHeight: number, currentHeight: number) { - return { id, properties: null, minHeight, currentHeight } + return { id, minHeight, currentHeight } } function removal( @@ -191,5 +191,5 @@ function removal( fromHeightInclusive: number, toHeightInclusive: number, ) { - return { id, properties: null, fromHeightInclusive, toHeightInclusive } + return { id, fromHeightInclusive, toHeightInclusive } } diff --git a/packages/uif/src/indexers/multi/diffConfigurations.ts b/packages/uif/src/indexers/multi/diffConfigurations.ts index 1286c053..3ddb654e 100644 --- a/packages/uif/src/indexers/multi/diffConfigurations.ts +++ b/packages/uif/src/indexers/multi/diffConfigurations.ts @@ -5,11 +5,11 @@ import { StoredConfiguration, } from './types' -export function diffConfigurations( - actual: Configuration[], - stored: StoredConfiguration[], +export function diffConfigurations( + actual: Configuration[], + stored: StoredConfiguration[], ): { - toRemove: RemovalConfiguration[] + toRemove: RemovalConfiguration[] safeHeight: number | null } { let safeHeight: number | null = Infinity @@ -18,11 +18,10 @@ export function diffConfigurations( const actualMap = new Map(actual.map((c) => [c.id, c])) const storedMap = new Map(stored.map((c) => [c.id, c])) - const toRemove: RemovalConfiguration[] = stored + const toRemove: RemovalConfiguration[] = stored .filter((c) => !actualMap.has(c.id)) .map((c) => ({ id: c.id, - properties: c.properties, fromHeightInclusive: c.minHeight, toHeightInclusive: c.currentHeight, })) @@ -51,14 +50,12 @@ export function diffConfigurations( // We will re-download everything from the beginning toRemove.push({ id: stored.id, - properties: stored.properties, fromHeightInclusive: stored.minHeight, toHeightInclusive: stored.currentHeight, }) } else if (stored.minHeight < c.minHeight) { toRemove.push({ id: stored.id, - properties: stored.properties, fromHeightInclusive: stored.minHeight, toHeightInclusive: c.minHeight - 1, }) @@ -67,7 +64,6 @@ export function diffConfigurations( if (c.maxHeight !== null && stored.currentHeight > c.maxHeight) { toRemove.push({ id: stored.id, - properties: stored.properties, fromHeightInclusive: c.maxHeight + 1, toHeightInclusive: stored.currentHeight, }) diff --git a/packages/uif/src/indexers/multi/types.ts b/packages/uif/src/indexers/multi/types.ts index e9b3072f..33c0fbe9 100644 --- a/packages/uif/src/indexers/multi/types.ts +++ b/packages/uif/src/indexers/multi/types.ts @@ -5,16 +5,14 @@ export interface Configuration { maxHeight: number | null } -export interface StoredConfiguration { +export interface StoredConfiguration { id: string - properties: T minHeight: number currentHeight: number } -export interface RemovalConfiguration { +export interface RemovalConfiguration { id: string - properties: T fromHeightInclusive: number toHeightInclusive: number } From 4c05d46765172bc006d2108b8946bf5000edb3f5 Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Thu, 21 Mar 2024 19:39:32 +0100 Subject: [PATCH 11/31] Start calling saveConfigurations --- .../src/indexers/multi/MultiIndexer.test.ts | 47 +++++++++-- .../uif/src/indexers/multi/MultiIndexer.ts | 83 +++++++++++++------ .../indexers/multi/diffConfigurations.test.ts | 53 ++++++------ .../src/indexers/multi/diffConfigurations.ts | 30 +++++-- packages/uif/src/indexers/multi/types.ts | 2 +- 5 files changed, 148 insertions(+), 67 deletions(-) diff --git a/packages/uif/src/indexers/multi/MultiIndexer.test.ts b/packages/uif/src/indexers/multi/MultiIndexer.test.ts index 9196c3d3..e4ca7e7f 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.test.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.test.ts @@ -2,7 +2,7 @@ import { Logger } from '@l2beat/backend-tools' import { expect, mockFn } from 'earl' import { IMultiIndexer, MultiIndexer } from './MultiIndexer' -import { Configuration, StoredConfiguration } from './types' +import { Configuration, SavedConfiguration } from './types' describe(MultiIndexer.name, () => { describe(MultiIndexer.prototype.update.name, () => { @@ -11,6 +11,7 @@ describe(MultiIndexer.name, () => { [actual('a', 100, 200), actual('b', 300, 400)], [], ) + await testIndexer.initialize() const newHeight = await testIndexer.update(100, 500) @@ -18,13 +19,17 @@ describe(MultiIndexer.name, () => { expect(testIndexer.multiUpdate).toHaveBeenOnlyCalledWith(100, 200, [ actual('a', 100, 200), ]) + expect(testIndexer.saveConfigurations).toHaveBeenOnlyCalledWith([ + saved('a', 100, 200), + ]) }) it('calls multiUpdate with a late matching configuration', async () => { const testIndexer = new TestMultiIndexer( [actual('a', 100, 200), actual('b', 300, 400)], - [], + [saved('a', 100, 200)], ) + await testIndexer.initialize() const newHeight = await testIndexer.update(300, 500) @@ -32,6 +37,10 @@ describe(MultiIndexer.name, () => { expect(testIndexer.multiUpdate).toHaveBeenOnlyCalledWith(300, 400, [ actual('b', 300, 400), ]) + expect(testIndexer.saveConfigurations).toHaveBeenOnlyCalledWith([ + saved('a', 100, 200), + saved('b', 300, 400), + ]) }) it('calls multiUpdate with a two matching configurations', async () => { @@ -39,6 +48,7 @@ describe(MultiIndexer.name, () => { [actual('a', 100, 200), actual('b', 100, 400)], [], ) + await testIndexer.initialize() const newHeight = await testIndexer.update(100, 500) @@ -47,13 +57,18 @@ describe(MultiIndexer.name, () => { actual('a', 100, 200), actual('b', 100, 400), ]) + expect(testIndexer.saveConfigurations).toHaveBeenOnlyCalledWith([ + saved('a', 100, 200), + saved('b', 100, 200), + ]) }) it('calls multiUpdate with a two middle matching configurations', async () => { const testIndexer = new TestMultiIndexer( [actual('a', 100, 400), actual('b', 200, 500)], - [], + [saved('a', 100, 300), saved('b', 200, 300)], ) + await testIndexer.initialize() const newHeight = await testIndexer.update(300, 600) @@ -62,6 +77,10 @@ describe(MultiIndexer.name, () => { actual('a', 100, 400), actual('b', 200, 500), ]) + expect(testIndexer.saveConfigurations).toHaveBeenOnlyCalledWith([ + saved('a', 100, 400), + saved('b', 200, 400), + ]) }) it('skips calling multiUpdate if we are too early', async () => { @@ -69,11 +88,13 @@ describe(MultiIndexer.name, () => { [actual('a', 100, 200), actual('b', 300, 400)], [], ) + await testIndexer.initialize() const newHeight = await testIndexer.update(null, 500) expect(newHeight).toEqual(99) expect(testIndexer.multiUpdate).not.toHaveBeenCalled() + expect(testIndexer.saveConfigurations).not.toHaveBeenCalled() }) it('skips calling multiUpdate if we are too late', async () => { @@ -81,11 +102,13 @@ describe(MultiIndexer.name, () => { [actual('a', 100, 200), actual('b', 300, 400)], [], ) + await testIndexer.initialize() const newHeight = await testIndexer.update(400, 500) expect(newHeight).toEqual(500) expect(testIndexer.multiUpdate).not.toHaveBeenCalled() + expect(testIndexer.saveConfigurations).not.toHaveBeenCalled() }) it('skips calling multiUpdate between configs', async () => { @@ -93,11 +116,13 @@ describe(MultiIndexer.name, () => { [actual('a', 100, 200), actual('b', 300, 400)], [], ) + await testIndexer.initialize() const newHeight = await testIndexer.update(200, 500) expect(newHeight).toEqual(299) expect(testIndexer.multiUpdate).not.toHaveBeenCalled() + expect(testIndexer.saveConfigurations).not.toHaveBeenCalled() }) }) }) @@ -108,25 +133,29 @@ class TestMultiIndexer { constructor( configurations: Configuration[], - private readonly _stored: StoredConfiguration[], + private readonly _saved: SavedConfiguration[], ) { super(Logger.SILENT, [], configurations) } - override multiInitialize(): Promise { - return Promise.resolve(this._stored) + override multiInitialize(): Promise { + return Promise.resolve(this._saved) } multiUpdate = mockFn['multiUpdate']>((_, targetHeight) => Promise.resolve(targetHeight), ) - removeData = mockFn['removeData']>() + removeData = mockFn['removeData']>().resolvesTo(undefined) - setStoredConfigurations = - mockFn['setStoredConfigurations']>() + saveConfigurations = + mockFn['saveConfigurations']>().resolvesTo(undefined) } function actual(id: string, minHeight: number, maxHeight: number | null) { return { id, properties: null, minHeight, maxHeight } } + +function saved(id: string, minHeight: number, currentHeight: number) { + return { id, minHeight, currentHeight } +} diff --git a/packages/uif/src/indexers/multi/MultiIndexer.ts b/packages/uif/src/indexers/multi/MultiIndexer.ts index e4babe43..ce0cece9 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.ts @@ -10,7 +10,7 @@ import { Configuration, ConfigurationRange, RemovalConfiguration, - StoredConfiguration, + SavedConfiguration, } from './types' export interface IMultiIndexer { @@ -23,7 +23,7 @@ export interface IMultiIndexer { * previously with `setStoredConfigurations`. It shouldn't call * `setStoredConfigurations` itself. */ - multiInitialize: () => Promise + multiInitialize: () => Promise /** * Removes data that was previously synced but because configurations changed @@ -42,9 +42,10 @@ export interface IMultiIndexer { * indexer can only fetch data up to 110 and return 110. The next time this * method will be called with `.update(110, 200, [...])`. * - * @param currentHeight The height that the indexer has synced up to previously. Can - * be `null` if no data was synced. This value is exclusive so the indexer - * should not fetch data for this height. + * @param currentHeight The height that the indexer has synced up to + * previously. This value is exclusive so the indexer should not fetch data + * for this height. If the indexer hasn't synced anything previously this + * will equal the minimum height of all configurations - 1. * * @param targetHeight The height that the indexer should sync up to. This value is * inclusive so the indexer should eventually fetch data for this height. @@ -56,19 +57,24 @@ export interface IMultiIndexer { * @returns The height that the indexer has synced up to. Returning * `currentHeight` means that the indexer has not synced any data. Returning * a value greater than `currentHeight` means that the indexer has synced up - * to that height. Returning a value less than `currentHeight` will trigger - * invalidation down to the returned value. Returning `null` will invalidate - * all data. Returning a value greater than `targetHeight` is not permitted. + * to that height. Returning a value less than `currentHeight` or greater than + * `targetHeight` is not permitted. */ multiUpdate: ( - currentHeight: number | null, + currentHeight: number, targetHeight: number, configurations: Configuration[], - ) => Promise + ) => Promise - setStoredConfigurations: ( - configurations: StoredConfiguration[], - ) => Promise + /** + * Saves configurations that the indexer should use to sync data. The + * configurations saved here should be read in the `multiInitialize` method. + * + * @param configurations The configurations that the indexer should save. The + * indexer should save the returned configurations and ensure that no other + * configurations are persisted. + */ + saveConfigurations: (configurations: SavedConfiguration[]) => Promise } export abstract class MultiIndexer @@ -76,6 +82,7 @@ export abstract class MultiIndexer implements IChildIndexer, IMultiIndexer { private readonly ranges: ConfigurationRange[] + private saved: SavedConfiguration[] = [] constructor( logger: Logger, @@ -90,24 +97,28 @@ export abstract class MultiIndexer this.ranges = toRanges(configurations) } - abstract multiInitialize(): Promise + abstract multiInitialize(): Promise abstract multiUpdate( - currentHeight: number | null, + currentHeight: number, targetHeight: number, configurations: Configuration[], - ): Promise + ): Promise abstract removeData(configurations: RemovalConfiguration[]): Promise - abstract setStoredConfigurations( - configurations: StoredConfiguration[], + abstract saveConfigurations( + configurations: SavedConfiguration[], ): Promise async initialize(): Promise { - const stored = await this.multiInitialize() - const { toRemove, safeHeight } = diffConfigurations( + const saved = await this.multiInitialize() + const { toRemove, toSave, safeHeight } = diffConfigurations( this.configurations, - stored, + saved, ) - await this.removeData(toRemove) + this.saved = toSave + if (toRemove.length > 0) { + await this.removeData(toRemove) + await this.saveConfigurations(toSave) + } return safeHeight } @@ -123,7 +134,12 @@ export abstract class MultiIndexer if (!range) { throw new Error('Programmer error, there should always be a range') } - if (range.configurations.length === 0) { + if ( + range.configurations.length === 0 || + // this check is only necessary for TypeScript. If currentHeight is null + // then the first condition will always be true + currentHeight === null + ) { return Height.min(range.to, targetHeight) } @@ -134,11 +150,28 @@ export abstract class MultiIndexer minTarget, range.configurations, ) - if (Height.gt(newHeight, minTarget)) { + if (newHeight < currentHeight || newHeight > minTarget) { throw new Error( - 'Programmer error, returned height cannot be greater than targetHeight', + 'Programmer error, returned height must be between currentHeight and targetHeight.', ) } + + if (newHeight > currentHeight) { + for (const configuration of range.configurations) { + const saved = this.saved.find((c) => c.id === configuration.id) + if (saved) { + saved.currentHeight = newHeight + } else { + this.saved.push({ + id: configuration.id, + minHeight: configuration.minHeight, + currentHeight: newHeight, + }) + } + } + await this.saveConfigurations(this.saved) + } + return newHeight } diff --git a/packages/uif/src/indexers/multi/diffConfigurations.test.ts b/packages/uif/src/indexers/multi/diffConfigurations.test.ts index dc38f508..b9c9694c 100644 --- a/packages/uif/src/indexers/multi/diffConfigurations.test.ts +++ b/packages/uif/src/indexers/multi/diffConfigurations.test.ts @@ -20,7 +20,7 @@ describe(diffConfigurations.name, () => { describe('regular sync', () => { it('empty actual and stored', () => { const result = diffConfigurations([], []) - expect(result).toEqual({ toRemove: [], safeHeight: Infinity }) + expect(result).toEqual({ toRemove: [], toSave: [], safeHeight: Infinity }) }) it('empty stored', () => { @@ -28,16 +28,17 @@ describe(diffConfigurations.name, () => { [actual('a', 100, null), actual('b', 200, 300)], [], ) - expect(result).toEqual({ toRemove: [], safeHeight: 99 }) + expect(result).toEqual({ toRemove: [], toSave: [], safeHeight: 99 }) }) it('partially synced, both early', () => { const result = diffConfigurations( [actual('a', 100, 400), actual('b', 200, null)], - [stored('a', 100, 300), stored('b', 200, 300)], + [saved('a', 100, 300), saved('b', 200, 300)], ) expect(result).toEqual({ toRemove: [], + toSave: [saved('a', 100, 300), saved('b', 200, 300)], safeHeight: 300, }) }) @@ -45,10 +46,11 @@ describe(diffConfigurations.name, () => { it('partially synced, one not yet started', () => { const result = diffConfigurations( [actual('a', 100, 400), actual('b', 555, null)], - [stored('a', 100, 300)], + [saved('a', 100, 300)], ) expect(result).toEqual({ toRemove: [], + toSave: [saved('a', 100, 300)], safeHeight: 300, }) }) @@ -56,10 +58,11 @@ describe(diffConfigurations.name, () => { it('partially synced, one finished', () => { const result = diffConfigurations( [actual('a', 100, 555), actual('b', 200, 300)], - [stored('a', 100, 400), stored('b', 200, 300)], + [saved('a', 100, 400), saved('b', 200, 300)], ) expect(result).toEqual({ toRemove: [], + toSave: [saved('a', 100, 400), saved('b', 200, 300)], safeHeight: 400, }) }) @@ -67,10 +70,11 @@ describe(diffConfigurations.name, () => { it('partially synced, one finished, one infinite', () => { const result = diffConfigurations( [actual('a', 100, null), actual('b', 200, 300)], - [stored('a', 100, 400), stored('b', 200, 300)], + [saved('a', 100, 400), saved('b', 200, 300)], ) expect(result).toEqual({ toRemove: [], + toSave: [saved('a', 100, 400), saved('b', 200, 300)], safeHeight: 400, }) }) @@ -78,10 +82,11 @@ describe(diffConfigurations.name, () => { it('both synced', () => { const result = diffConfigurations( [actual('a', 100, 400), actual('b', 200, 300)], - [stored('a', 100, 400), stored('b', 200, 300)], + [saved('a', 100, 400), saved('b', 200, 300)], ) expect(result).toEqual({ toRemove: [], + toSave: [saved('a', 100, 400), saved('b', 200, 300)], safeHeight: Infinity, }) }) @@ -91,10 +96,11 @@ describe(diffConfigurations.name, () => { it('empty actual', () => { const result = diffConfigurations( [], - [stored('a', 100, 300), stored('b', 200, 300)], + [saved('a', 100, 300), saved('b', 200, 300)], ) expect(result).toEqual({ toRemove: [removal('a', 100, 300), removal('b', 200, 300)], + toSave: [], safeHeight: Infinity, }) }) @@ -102,21 +108,11 @@ describe(diffConfigurations.name, () => { it('single removed', () => { const result = diffConfigurations( [actual('b', 200, 400)], - [stored('a', 100, 300), stored('b', 200, 300)], - ) - expect(result).toEqual({ - toRemove: [removal('a', 100, 300)], - safeHeight: 300, - }) - }) - - it('single removed', () => { - const result = diffConfigurations( - [actual('b', 200, 400)], - [stored('a', 100, 300), stored('b', 200, 300)], + [saved('a', 100, 300), saved('b', 200, 300)], ) expect(result).toEqual({ toRemove: [removal('a', 100, 300)], + toSave: [saved('b', 200, 300)], safeHeight: 300, }) }) @@ -124,10 +120,11 @@ describe(diffConfigurations.name, () => { it('maxHeight updated up', () => { const result = diffConfigurations( [actual('a', 100, 400)], - [stored('a', 100, 300)], + [saved('a', 100, 300)], ) expect(result).toEqual({ toRemove: [], + toSave: [saved('a', 100, 300)], safeHeight: 300, }) }) @@ -135,10 +132,11 @@ describe(diffConfigurations.name, () => { it('maxHeight updated down', () => { const result = diffConfigurations( [actual('a', 100, 200)], - [stored('a', 100, 300)], + [saved('a', 100, 300)], ) expect(result).toEqual({ toRemove: [removal('a', 201, 300)], + toSave: [saved('a', 100, 200)], safeHeight: Infinity, }) }) @@ -146,10 +144,11 @@ describe(diffConfigurations.name, () => { it('minHeight updated up', () => { const result = diffConfigurations( [actual('a', 200, 400)], - [stored('a', 100, 300)], + [saved('a', 100, 300)], ) expect(result).toEqual({ toRemove: [removal('a', 100, 199)], + toSave: [saved('a', 200, 300)], safeHeight: 300, }) }) @@ -157,10 +156,11 @@ describe(diffConfigurations.name, () => { it('minHeight updated down', () => { const result = diffConfigurations( [actual('a', 100, 400)], - [stored('a', 200, 300)], + [saved('a', 200, 300)], ) expect(result).toEqual({ toRemove: [removal('a', 200, 300)], + toSave: [], safeHeight: 99, }) }) @@ -168,10 +168,11 @@ describe(diffConfigurations.name, () => { it('both min and max height updated', () => { const result = diffConfigurations( [actual('a', 200, 300)], - [stored('a', 100, 400)], + [saved('a', 100, 400)], ) expect(result).toEqual({ toRemove: [removal('a', 100, 199), removal('a', 301, 400)], + toSave: [saved('a', 200, 300)], safeHeight: Infinity, }) }) @@ -182,7 +183,7 @@ function actual(id: string, minHeight: number, maxHeight: number | null) { return { id, properties: null, minHeight, maxHeight } } -function stored(id: string, minHeight: number, currentHeight: number) { +function saved(id: string, minHeight: number, currentHeight: number) { return { id, minHeight, currentHeight } } diff --git a/packages/uif/src/indexers/multi/diffConfigurations.ts b/packages/uif/src/indexers/multi/diffConfigurations.ts index 3ddb654e..7c71e5fb 100644 --- a/packages/uif/src/indexers/multi/diffConfigurations.ts +++ b/packages/uif/src/indexers/multi/diffConfigurations.ts @@ -2,23 +2,24 @@ import { Height } from '../../height' import { Configuration, RemovalConfiguration, - StoredConfiguration, + SavedConfiguration, } from './types' export function diffConfigurations( actual: Configuration[], - stored: StoredConfiguration[], + saved: SavedConfiguration[], ): { toRemove: RemovalConfiguration[] + toSave: SavedConfiguration[] safeHeight: number | null } { let safeHeight: number | null = Infinity const knownIds = new Set() const actualMap = new Map(actual.map((c) => [c.id, c])) - const storedMap = new Map(stored.map((c) => [c.id, c])) + const savedMap = new Map(saved.map((c) => [c.id, c])) - const toRemove: RemovalConfiguration[] = stored + const toRemove: RemovalConfiguration[] = saved .filter((c) => !actualMap.has(c.id)) .map((c) => ({ id: c.id, @@ -38,7 +39,7 @@ export function diffConfigurations( ) } - const stored = storedMap.get(c.id) + const stored = savedMap.get(c.id) if (!stored) { safeHeight = Height.min(safeHeight, c.minHeight - 1) continue @@ -75,5 +76,22 @@ export function diffConfigurations( } } - return { toRemove, safeHeight } + const toSave = saved + .map((c): SavedConfiguration | undefined => { + const actual = actualMap.get(c.id) + if (!actual || actual.minHeight < c.minHeight) { + return undefined + } + return { + id: c.id, + minHeight: actual.minHeight, + currentHeight: + actual.maxHeight === null + ? c.currentHeight + : Math.min(c.currentHeight, actual.maxHeight), + } + }) + .filter((c): c is SavedConfiguration => c !== undefined) + + return { toRemove, toSave, safeHeight } } diff --git a/packages/uif/src/indexers/multi/types.ts b/packages/uif/src/indexers/multi/types.ts index 33c0fbe9..10945378 100644 --- a/packages/uif/src/indexers/multi/types.ts +++ b/packages/uif/src/indexers/multi/types.ts @@ -5,7 +5,7 @@ export interface Configuration { maxHeight: number | null } -export interface StoredConfiguration { +export interface SavedConfiguration { id: string minHeight: number currentHeight: number From efaf191183e6d0224f2e6aa126888fe85df03d5f Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Thu, 21 Mar 2024 19:56:32 +0100 Subject: [PATCH 12/31] Add more tests for MultiIndexer --- .../src/indexers/multi/MultiIndexer.test.ts | 119 ++++++++++++++++++ .../uif/src/indexers/multi/MultiIndexer.ts | 4 +- 2 files changed, 121 insertions(+), 2 deletions(-) diff --git a/packages/uif/src/indexers/multi/MultiIndexer.test.ts b/packages/uif/src/indexers/multi/MultiIndexer.test.ts index e4ca7e7f..48540b4b 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.test.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.test.ts @@ -5,6 +5,39 @@ import { IMultiIndexer, MultiIndexer } from './MultiIndexer' import { Configuration, SavedConfiguration } from './types' describe(MultiIndexer.name, () => { + describe(MultiIndexer.prototype.initialize.name, () => { + it('calls multiInitialize and saves configurations', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 400), actual('b', 200, 500)], + [saved('a', 100, 300), saved('b', 200, 300), saved('c', 100, 300)], + ) + + const newHeight = await testIndexer.initialize() + expect(newHeight).toEqual(300) + + expect(testIndexer.removeData).toHaveBeenOnlyCalledWith([ + removal('c', 100, 300), + ]) + expect(testIndexer.saveConfigurations).toHaveBeenOnlyCalledWith([ + saved('a', 100, 300), + saved('b', 200, 300), + ]) + }) + + it('skips calling removeData if there is nothing to remove', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 400), actual('b', 200, 500)], + [saved('a', 100, 400), saved('b', 200, 500)], + ) + + const newHeight = await testIndexer.initialize() + expect(newHeight).toEqual(Infinity) + + expect(testIndexer.removeData).not.toHaveBeenCalled() + expect(testIndexer.saveConfigurations).not.toHaveBeenCalled() + }) + }) + describe(MultiIndexer.prototype.update.name, () => { it('calls multiUpdate with an early matching configuration', async () => { const testIndexer = new TestMultiIndexer( @@ -125,6 +158,84 @@ describe(MultiIndexer.name, () => { expect(testIndexer.saveConfigurations).not.toHaveBeenCalled() }) }) + + describe('multiUpdate', () => { + it('returns the currentHeight', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 300), actual('b', 100, 400)], + [saved('a', 100, 200), saved('b', 100, 200)], + ) + await testIndexer.initialize() + + testIndexer.multiUpdate.resolvesTo(200) + + const newHeight = await testIndexer.update(200, 500) + expect(newHeight).toEqual(200) + expect(testIndexer.saveConfigurations).not.toHaveBeenCalled() + }) + + it('returns the targetHeight', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 300), actual('b', 100, 400)], + [saved('a', 100, 200), saved('b', 100, 200)], + ) + await testIndexer.initialize() + + testIndexer.multiUpdate.resolvesTo(300) + + const newHeight = await testIndexer.update(200, 300) + expect(newHeight).toEqual(300) + expect(testIndexer.saveConfigurations).toHaveBeenOnlyCalledWith([ + saved('a', 100, 300), + saved('b', 100, 300), + ]) + }) + + it('returns something in between', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 300), actual('b', 100, 400)], + [saved('a', 100, 200), saved('b', 100, 200)], + ) + await testIndexer.initialize() + + testIndexer.multiUpdate.resolvesTo(250) + + const newHeight = await testIndexer.update(200, 300) + expect(newHeight).toEqual(250) + expect(testIndexer.saveConfigurations).toHaveBeenOnlyCalledWith([ + saved('a', 100, 250), + saved('b', 100, 250), + ]) + }) + + it('cannot return less than currentHeight', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 300), actual('b', 100, 400)], + [saved('a', 100, 200), saved('b', 100, 200)], + ) + await testIndexer.initialize() + + testIndexer.multiUpdate.resolvesTo(150) + + await expect(testIndexer.update(200, 300)).toBeRejectedWith( + /returned height must be between currentHeight and targetHeight/, + ) + }) + + it('cannot return more than targetHeight', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 300), actual('b', 100, 400)], + [saved('a', 100, 200), saved('b', 100, 200)], + ) + await testIndexer.initialize() + + testIndexer.multiUpdate.resolvesTo(350) + + await expect(testIndexer.update(200, 300)).toBeRejectedWith( + /returned height must be between currentHeight and targetHeight/, + ) + }) + }) }) class TestMultiIndexer @@ -159,3 +270,11 @@ function actual(id: string, minHeight: number, maxHeight: number | null) { function saved(id: string, minHeight: number, currentHeight: number) { return { id, minHeight, currentHeight } } + +function removal( + id: string, + fromHeightInclusive: number, + toHeightInclusive: number, +) { + return { id, fromHeightInclusive, toHeightInclusive } +} diff --git a/packages/uif/src/indexers/multi/MultiIndexer.ts b/packages/uif/src/indexers/multi/MultiIndexer.ts index ce0cece9..5ba4cc72 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.ts @@ -31,8 +31,8 @@ export interface IMultiIndexer { * in each configuration. It is possible for multiple ranges to share a * configuration id! * - * This method is only called during the initialization of the indexer, after - * `multiInitialize` returns. + * This method can only be called during the initialization of the indexer, + * after `multiInitialize` returns. */ removeData: (configurations: RemovalConfiguration[]) => Promise From 536c529cb9ba50d39ea8460d2e0a33aa87c9439f Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Thu, 21 Mar 2024 22:50:48 +0100 Subject: [PATCH 13/31] MultiIndexer should now be feature complete --- .../src/indexers/multi/MultiIndexer.test.ts | 84 +++++++++++++-- .../uif/src/indexers/multi/MultiIndexer.ts | 100 +++++++++++++----- packages/uif/src/indexers/multi/types.ts | 4 + 3 files changed, 152 insertions(+), 36 deletions(-) diff --git a/packages/uif/src/indexers/multi/MultiIndexer.test.ts b/packages/uif/src/indexers/multi/MultiIndexer.test.ts index 48540b4b..e56238c6 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.test.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.test.ts @@ -50,7 +50,7 @@ describe(MultiIndexer.name, () => { expect(newHeight).toEqual(200) expect(testIndexer.multiUpdate).toHaveBeenOnlyCalledWith(100, 200, [ - actual('a', 100, 200), + update('a', 100, 200, false), ]) expect(testIndexer.saveConfigurations).toHaveBeenOnlyCalledWith([ saved('a', 100, 200), @@ -68,7 +68,7 @@ describe(MultiIndexer.name, () => { expect(newHeight).toEqual(400) expect(testIndexer.multiUpdate).toHaveBeenOnlyCalledWith(300, 400, [ - actual('b', 300, 400), + update('b', 300, 400, false), ]) expect(testIndexer.saveConfigurations).toHaveBeenOnlyCalledWith([ saved('a', 100, 200), @@ -76,7 +76,7 @@ describe(MultiIndexer.name, () => { ]) }) - it('calls multiUpdate with a two matching configurations', async () => { + it('calls multiUpdate with two matching configurations', async () => { const testIndexer = new TestMultiIndexer( [actual('a', 100, 200), actual('b', 100, 400)], [], @@ -87,8 +87,8 @@ describe(MultiIndexer.name, () => { expect(newHeight).toEqual(200) expect(testIndexer.multiUpdate).toHaveBeenOnlyCalledWith(100, 200, [ - actual('a', 100, 200), - actual('b', 100, 400), + update('a', 100, 200, false), + update('b', 100, 400, false), ]) expect(testIndexer.saveConfigurations).toHaveBeenOnlyCalledWith([ saved('a', 100, 200), @@ -96,7 +96,7 @@ describe(MultiIndexer.name, () => { ]) }) - it('calls multiUpdate with a two middle matching configurations', async () => { + it('calls multiUpdate with two middle matching configurations', async () => { const testIndexer = new TestMultiIndexer( [actual('a', 100, 400), actual('b', 200, 500)], [saved('a', 100, 300), saved('b', 200, 300)], @@ -107,8 +107,8 @@ describe(MultiIndexer.name, () => { expect(newHeight).toEqual(400) expect(testIndexer.multiUpdate).toHaveBeenOnlyCalledWith(300, 400, [ - actual('a', 100, 400), - actual('b', 200, 500), + update('a', 100, 400, false), + update('b', 200, 500, false), ]) expect(testIndexer.saveConfigurations).toHaveBeenOnlyCalledWith([ saved('a', 100, 400), @@ -157,6 +157,65 @@ describe(MultiIndexer.name, () => { expect(testIndexer.multiUpdate).not.toHaveBeenCalled() expect(testIndexer.saveConfigurations).not.toHaveBeenCalled() }) + + it('calls multiUpdate with a matching configuration with data', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 200), actual('b', 100, 400)], + [saved('a', 100, 200)], + ) + await testIndexer.initialize() + + const newHeight = await testIndexer.update(100, 500) + + expect(newHeight).toEqual(200) + expect(testIndexer.multiUpdate).toHaveBeenOnlyCalledWith(100, 200, [ + update('a', 100, 200, true), + update('b', 100, 400, false), + ]) + expect(testIndexer.saveConfigurations).toHaveBeenOnlyCalledWith([ + saved('a', 100, 200), + saved('b', 100, 200), + ]) + }) + + it('multiple update calls', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 200), actual('b', 100, 400)], + [saved('a', 100, 200)], + ) + await testIndexer.initialize() + + expect(await testIndexer.update(100, 500)).toEqual(200) + expect(testIndexer.multiUpdate).toHaveBeenNthCalledWith(1, 100, 200, [ + update('a', 100, 200, true), + update('b', 100, 400, false), + ]) + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(1, [ + saved('a', 100, 200), + saved('b', 100, 200), + ]) + + // The same range. In real life might be a result of a parent reorg + expect(await testIndexer.update(100, 500)).toEqual(200) + expect(testIndexer.multiUpdate).toHaveBeenNthCalledWith(2, 100, 200, [ + update('a', 100, 200, true), + update('b', 100, 400, true), + ]) + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(2, [ + saved('a', 100, 200), + saved('b', 100, 200), + ]) + + // Next range + expect(await testIndexer.update(200, 500)).toEqual(400) + expect(testIndexer.multiUpdate).toHaveBeenNthCalledWith(3, 200, 400, [ + update('b', 100, 400, false), + ]) + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(3, [ + saved('a', 100, 200), + saved('b', 100, 400), + ]) + }) }) describe('multiUpdate', () => { @@ -271,6 +330,15 @@ function saved(id: string, minHeight: number, currentHeight: number) { return { id, minHeight, currentHeight } } +function update( + id: string, + minHeight: number, + maxHeight: number | null, + hasData: boolean, +) { + return { id, properties: null, minHeight, maxHeight, hasData } +} + function removal( id: string, fromHeightInclusive: number, diff --git a/packages/uif/src/indexers/multi/MultiIndexer.ts b/packages/uif/src/indexers/multi/MultiIndexer.ts index 5ba4cc72..3e094ce9 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.ts @@ -11,6 +11,7 @@ import { ConfigurationRange, RemovalConfiguration, SavedConfiguration, + UpdateConfiguration, } from './types' export interface IMultiIndexer { @@ -52,7 +53,9 @@ export interface IMultiIndexer { * * @param configurations The configurations that the indexer should use to * sync data. The configurations are guaranteed to be in the range of - * `currentHeight` and `targetHeight`. + * `currentHeight` and `targetHeight`. Some of those configurations might + * have been synced previously for this range. Those configurations + * will include the `hasData` flag set to `true`. * * @returns The height that the indexer has synced up to. Returning * `currentHeight` means that the indexer has not synced any data. Returning @@ -63,7 +66,7 @@ export interface IMultiIndexer { multiUpdate: ( currentHeight: number, targetHeight: number, - configurations: Configuration[], + configurations: UpdateConfiguration[], ) => Promise /** @@ -101,7 +104,7 @@ export abstract class MultiIndexer abstract multiUpdate( currentHeight: number, targetHeight: number, - configurations: Configuration[], + configurations: UpdateConfiguration[], ): Promise abstract removeData(configurations: RemovalConfiguration[]): Promise abstract saveConfigurations( @@ -126,49 +129,36 @@ export abstract class MultiIndexer currentHeight: number | null, targetHeight: number, ): Promise { - const range = this.ranges.find( - (range) => - currentHeight === null || - (range.from <= currentHeight + 1 && range.to > currentHeight), - ) - if (!range) { - throw new Error('Programmer error, there should always be a range') - } + const range = findRange(this.ranges, currentHeight) if ( range.configurations.length === 0 || - // this check is only necessary for TypeScript. If currentHeight is null + // This check is only necessary for TypeScript. If currentHeight is null // then the first condition will always be true currentHeight === null ) { return Height.min(range.to, targetHeight) } - const minTarget = Math.min(range.to, targetHeight) + const { configurations, minCurrentHeight } = getConfigurationsInRange( + range, + this.saved, + currentHeight, + ) + const minTargetHeight = Math.min(range.to, targetHeight, minCurrentHeight) const newHeight = await this.multiUpdate( currentHeight, - minTarget, - range.configurations, + minTargetHeight, + configurations, ) - if (newHeight < currentHeight || newHeight > minTarget) { + if (newHeight < currentHeight || newHeight > minTargetHeight) { throw new Error( 'Programmer error, returned height must be between currentHeight and targetHeight.', ) } if (newHeight > currentHeight) { - for (const configuration of range.configurations) { - const saved = this.saved.find((c) => c.id === configuration.id) - if (saved) { - saved.currentHeight = newHeight - } else { - this.saved.push({ - id: configuration.id, - minHeight: configuration.minHeight, - currentHeight: newHeight, - }) - } - } + updateSavedConfigurations(this.saved, configurations, newHeight) await this.saveConfigurations(this.saved) } @@ -183,3 +173,57 @@ export abstract class MultiIndexer return Promise.resolve() } } + +function findRange( + ranges: ConfigurationRange[], + currentHeight: number | null, +): ConfigurationRange { + const range = ranges.find( + (range) => + currentHeight === null || + (range.from <= currentHeight + 1 && range.to > currentHeight), + ) + if (!range) { + throw new Error('Programmer error, there should always be a range') + } + return range +} + +function getConfigurationsInRange( + range: ConfigurationRange, + savedConfigurations: SavedConfiguration[], + currentHeight: number, +): { configurations: UpdateConfiguration[]; minCurrentHeight: number } { + let minCurrentHeight = Infinity + const configurations = range.configurations.map( + (configuration): UpdateConfiguration => { + const saved = savedConfigurations.find((c) => c.id === configuration.id) + if (saved && saved.currentHeight > currentHeight) { + minCurrentHeight = Math.min(minCurrentHeight, saved.currentHeight) + return { ...configuration, hasData: true } + } else { + return { ...configuration, hasData: false } + } + }, + ) + return { configurations, minCurrentHeight } +} + +function updateSavedConfigurations( + savedConfigurations: SavedConfiguration[], + updatedConfigurations: UpdateConfiguration[], + newHeight: number, +): void { + for (const updated of updatedConfigurations) { + const saved = savedConfigurations.find((c) => c.id === updated.id) + if (saved) { + saved.currentHeight = newHeight + } else { + savedConfigurations.push({ + id: updated.id, + minHeight: updated.minHeight, + currentHeight: newHeight, + }) + } + } +} diff --git a/packages/uif/src/indexers/multi/types.ts b/packages/uif/src/indexers/multi/types.ts index 10945378..a372ad78 100644 --- a/packages/uif/src/indexers/multi/types.ts +++ b/packages/uif/src/indexers/multi/types.ts @@ -5,6 +5,10 @@ export interface Configuration { maxHeight: number | null } +export interface UpdateConfiguration extends Configuration { + hasData: boolean +} + export interface SavedConfiguration { id: string minHeight: number From 9fede21340f3c70f7028e3a6cdd7c5db600cd6d6 Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Thu, 21 Mar 2024 23:03:13 +0100 Subject: [PATCH 14/31] Make the API nicer sacrificing safety --- packages/uif/src/BaseIndexer.ts | 37 +- packages/uif/src/indexers/ChildIndexer.ts | 80 +--- packages/uif/src/indexers/RootIndexer.ts | 37 +- .../uif/src/indexers/SliceIndexer.test.ts | 344 ------------------ packages/uif/src/indexers/SliceIndexer.ts | 122 ------- .../src/indexers/multi/MultiIndexer.test.ts | 7 +- .../uif/src/indexers/multi/MultiIndexer.ts | 78 ++-- 7 files changed, 53 insertions(+), 652 deletions(-) delete mode 100644 packages/uif/src/indexers/SliceIndexer.test.ts delete mode 100644 packages/uif/src/indexers/SliceIndexer.ts diff --git a/packages/uif/src/BaseIndexer.ts b/packages/uif/src/BaseIndexer.ts index 96a960d3..e6234f6e 100644 --- a/packages/uif/src/BaseIndexer.ts +++ b/packages/uif/src/BaseIndexer.ts @@ -34,15 +34,16 @@ export abstract class BaseIndexer { /** * Initializes the indexer. It should return a height that the indexer has * synced up to. If the indexer has not synced any data, it should return - * `null`. + * `null`. For root indexers it should return the initial target height for + * the entire system. * * This method is expected to read the height that was saved previously with * `setSafeHeight`. It shouldn't call `setSafeHeight` itself. * - * For root indexers this method should also schedule a process to request - * ticks. For example with `setInterval(() => this.requestTick(), 1000)`. - * Since a root indexer probably doesn't save the height to a database, it - * can `return this.tick()` instead. + * For root indexers if `setSafeHeight` is implemented it should return the + * height that was saved previously. If not it can `return this.tick()`. + * This method should also schedule a process to request ticks. For example + * with `setInterval(() => this.requestTick(), 1000)`. */ abstract initialize(): Promise @@ -53,16 +54,16 @@ export abstract class BaseIndexer { * * When `initialize` is called it is expected that it will read the same * height that was saved here. + * + * Optional in root indexers. */ abstract setSafeHeight(height: number | null): Promise /** - * This method should only be implemented for a child indexer. - * - * It is responsible for the main data fetching process. It is up to the - * indexer to decide how much data to fetch. For example given - * `.update(100, 200)`, the indexer can only fetch data up to 110 and return - * 110. The next time this method will be called with `.update(110, 200)`. + * Implements the main data fetching process. It is up to the indexer to + * decide how much data to fetch. For example given `.update(100, 200)`, the + * indexer can only fetch data up to 110 and return 110. The next time this + * method will be called with `.update(110, 200)`. * * @param currentHeight The height that the indexer has synced up to previously. Can * be `null` if no data was synced. This value is exclusive so the indexer @@ -84,9 +85,7 @@ export abstract class BaseIndexer { ): Promise /** - * This method should only be implemented for a child indexer. - * - * It is responsible for invalidating data that was synced previously. It is + * Responsible for invalidating data that was synced previously. It is * possible that no data was synced and this method is still called. * * Invalidation can, but doesn't have to remove data from the database. If @@ -108,13 +107,11 @@ export abstract class BaseIndexer { abstract invalidate(targetHeight: number | null): Promise /** - * This method should only be implemented for a root indexer. - * - * It is responsible for providing the target height for the entire system. - * Some good examples of this are: the current time or the last block number. + * This method is responsible for providing the target height for the entire + * system. Some candidates for this are: the current time or the latest block + * number. * - * As opposed to `update` and `invalidate`, this method cannot return - * `null`. + * This method cannot return `null`. */ abstract tick(): Promise diff --git a/packages/uif/src/indexers/ChildIndexer.ts b/packages/uif/src/indexers/ChildIndexer.ts index 6ce20340..eedaad97 100644 --- a/packages/uif/src/indexers/ChildIndexer.ts +++ b/packages/uif/src/indexers/ChildIndexer.ts @@ -1,84 +1,6 @@ import { BaseIndexer } from '../BaseIndexer' -/** - * Because of the way TypeScript works, all child indexers need to - * `extends ChildIndexer` and `implements IChildIndexer`. Otherwise it - * is possible to have incorrect method signatures and TypeScript won't - * catch it. - */ -export interface IChildIndexer { - /** - * Initializes the indexer. It should return a height that the indexer has - * synced up to. If the indexer has not synced any data, it should return - * `null`. - * - * This method is expected to read the height that was saved previously with - * `setSafeHeight`. It shouldn't call `setSafeHeight` itself. - */ - initialize: () => Promise - - /** - * Saves the height (most likely to a database). The height given is the - * smallest height from all parents and what the indexer itself synced to - * previously. It can be `null`. - * - * When `initialize` is called it is expected that it will read the same - * height that was saved here. - */ - setSafeHeight: (height: number | null) => Promise - - /** - * Implements the main data fetching process. It is up to the indexer to - * decide how much data to fetch. For example given `.update(100, 200)`, the - * indexer can only fetch data up to 110 and return 110. The next time this - * method will be called with `.update(110, 200)`. - * - * @param currentHeight The height that the indexer has synced up to previously. Can - * be `null` if no data was synced. This value is exclusive so the indexer - * should not fetch data for this height. - * - * @param targetHeight The height that the indexer should sync up to. This value is - * inclusive so the indexer should eventually fetch data for this height. - * - * @returns The height that the indexer has synced up to. Returning - * `currentHeight` means that the indexer has not synced any data. Returning - * a value greater than `currentHeight` means that the indexer has synced up - * to that height. Returning a value less than `currentHeight` will trigger - * invalidation down to the returned value. Returning `null` will invalidate - * all data. Returning a value greater than `targetHeight` is not permitted. - */ - update: ( - currentHeight: number | null, - targetHeight: number, - ) => Promise - - /** - * Responsible for invalidating data that was synced previously. It is - * possible that no data was synced and this method is still called. - * - * Invalidation can, but doesn't have to remove data from the database. If - * you only want to rely on the safe height, you can just return the target - * height and the system will take care of the rest. - * - * This method doesn't have to invalidate all data. If you want to do it in - * steps, you can return a height that is larger than the target height. - * - * @param targetHeight The height that the indexer should invalidate down to. - * Can be `null`. If it is `null`, the indexer should invalidate all - * data. - * - * @returns The height that the indexer has invalidated down to. Returning - * `targetHeight` means that the indexer has invalidated all the required - * data. Returning a value greater than `targetHeight` means that the indexer - * has invalidated down to that height. - */ - invalidate: (targetHeight: number | null) => Promise -} - -export abstract class ChildIndexer - extends BaseIndexer - implements IChildIndexer -{ +export abstract class ChildIndexer extends BaseIndexer { override async tick(): Promise { return Promise.reject(new Error('ChildIndexer cannot tick')) } diff --git a/packages/uif/src/indexers/RootIndexer.ts b/packages/uif/src/indexers/RootIndexer.ts index 2febb4b8..bae51072 100644 --- a/packages/uif/src/indexers/RootIndexer.ts +++ b/packages/uif/src/indexers/RootIndexer.ts @@ -2,43 +2,8 @@ import { Logger } from '@l2beat/backend-tools' import { BaseIndexer } from '../BaseIndexer' import { RetryStrategy } from '../Retries' -/** - * Because of the way TypeScript works, all child indexers need to - * `extends RootIndexer` and `implements IRootIndexer`. Otherwise it - * is possible to have incorrect method signatures and TypeScript won't - * catch it. - */ -export interface IRootIndexer { - /** - * Initializes the indexer and returns the initial target height for the - * entire system. If `setSafeHeight` is implemented it should return the - * height that was saved previously. If not it can `return this.tick()`. - * - * This method should also schedule a process to request - * ticks. For example with `setInterval(() => this.requestTick(), 1000)`. - */ - initialize: () => Promise - /** - * This method is responsible for providing the target height for the entire - * system. Some candidates for this are: the current time or the latest block - * number. - * - * This method cannot return `null`. - */ - tick: () => Promise - - /** - * An optional method for saving the height (most likely to a database). The - * height can be `null`. - * - * When `initialize` is called it is expected that it will read the same - * height that was saved here. - */ - setSafeHeight?: (height: number | null) => Promise -} - -export abstract class RootIndexer extends BaseIndexer implements IRootIndexer { +export abstract class RootIndexer extends BaseIndexer { constructor(logger: Logger, opts?: { tickRetryStrategy?: RetryStrategy }) { super(logger, [], opts) } diff --git a/packages/uif/src/indexers/SliceIndexer.test.ts b/packages/uif/src/indexers/SliceIndexer.test.ts deleted file mode 100644 index d2d9d213..00000000 --- a/packages/uif/src/indexers/SliceIndexer.test.ts +++ /dev/null @@ -1,344 +0,0 @@ -import { Logger } from '@l2beat/backend-tools' -import { expect, mockFn } from 'earl' - -import { BaseIndexer } from '../BaseIndexer' -import { TestRootIndexer, waitUntil } from '../BaseIndexer.test' -import { IndexerAction } from '../reducer/types/IndexerAction' -import { RetryStrategy } from '../Retries' -import { - diffSlices, - SliceHash, - SliceIndexer, - SliceState, - SliceUpdate, -} from './SliceIndexer' - -describe(SliceIndexer.name, () => { - it('updates multiple slices', async () => { - const repository = testRepositoryFactory() - - const rootIndexer = new TestRootIndexer(0) - const sliceIndexer = new TestSliceIndexer( - ['token-a', 'token-b', 'token-c'], - repository, - [rootIndexer], - ) - - await rootIndexer.start() - await sliceIndexer.start() - - await sliceIndexer.finishInvalidate(0) - await rootIndexer.doTick(1) - await rootIndexer.finishTick(1) - - expect(repository.getSliceData('token-a').get(1)).toEqual(2) - expect(repository.getSliceData('token-b').get(1)).toEqual(2) - expect(repository.getSliceData('token-c').get(1)).toEqual(2) - - await rootIndexer.doTick(2) - await rootIndexer.finishTick(2) - - expect(repository.getSliceData('token-a').get(2)).toEqual(4) - expect(repository.getSliceData('token-b').get(2)).toEqual(4) - expect(repository.getSliceData('token-c').get(2)).toEqual(4) - }) - it('syncs new token after restart', async () => { - const repository = testRepositoryFactory() - - const rootIndexer = new TestRootIndexer(0) - const sliceIndexer = new TestSliceIndexer( - ['token-a', 'token-b', 'token-c'], - repository, - [rootIndexer], - ) - - await rootIndexer.start() - await sliceIndexer.start() - - await sliceIndexer.finishInvalidate(0) - await rootIndexer.doTick(1) - await rootIndexer.finishTick(1) - - expect(repository.getSliceData('token-a').get(1)).toEqual(2) - expect(repository.getSliceData('token-b').get(1)).toEqual(2) - expect(repository.getSliceData('token-c').get(1)).toEqual(2) - - // creating new instances to simulate restart - const newRootIndexer = new TestRootIndexer(1) - const newSliceIndexer = new TestSliceIndexer( - ['token-a', 'token-b', 'token-c', 'token-d'], - repository, - [newRootIndexer], - ) - - await newRootIndexer.start() - await newSliceIndexer.start() - - await newSliceIndexer.finishInvalidate(0) - - expect(repository.getSliceData('token-d').get(1)).toEqual(2) - }) - it('drops old token after restart', async () => { - const repository = testRepositoryFactory() - - const rootIndexer = new TestRootIndexer(0) - const sliceIndexer = new TestSliceIndexer( - ['token-a', 'token-b', 'token-c'], - repository, - [rootIndexer], - ) - - await rootIndexer.start() - await sliceIndexer.start() - - await sliceIndexer.finishInvalidate(0) - await rootIndexer.doTick(1) - await rootIndexer.finishTick(1) - - expect(repository.getSliceData('token-a').get(1)).toEqual(2) - expect(repository.getSliceData('token-b').get(1)).toEqual(2) - expect(repository.getSliceData('token-c').get(1)).toEqual(2) - - // creating new instances to simulate restart - const newRootIndexer = new TestRootIndexer(1) - const newSliceIndexer = new TestSliceIndexer( - ['token-a', 'token-b'], - repository, - [newRootIndexer], - ) - - await newRootIndexer.start() - await newSliceIndexer.start() - - await newSliceIndexer.finishInvalidate(0) - - expect(repository.removeSlices).toHaveBeenCalledWith(['token-c']) - expect(repository.getSliceData('token-c').get(1)).toBeNullish() - }) -}) -describe(diffSlices.name, () => { - describe('height increase', () => { - it('marks token-a to update', () => { - const expectedSlices: SliceHash[] = ['token-a', 'token-b', 'token-c'] - const actualSlices: SliceState[] = [ - { sliceHash: 'token-a', height: 100 }, - { sliceHash: 'token-b', height: 200 }, - { sliceHash: 'token-c', height: 300 }, - ] - const from = 100 - const to = 200 - - const { toRemove, toUpdate } = diffSlices( - expectedSlices, - actualSlices, - from, - to, - ) - - expect(toRemove).toEqual([]) - expect(toUpdate).toEqual([{ sliceHash: 'token-a', from, to }]) - }) - }) - describe('height decrease', () => { - it('marks token-a to update', () => { - const expectedSlices: SliceHash[] = ['token-a', 'token-b', 'token-c'] - const actualSlices: SliceState[] = [ - { sliceHash: 'token-a', height: 100 }, - { sliceHash: 'token-b', height: 200 }, - { sliceHash: 'token-c', height: 300 }, - ] - const from = 300 - const to = 200 - - const { toUpdate, toRemove } = diffSlices( - expectedSlices, - actualSlices, - from, - to, - ) - - expect(toRemove).toEqual([]) - expect(toUpdate).toEqual([{ sliceHash: 'token-a', from, to }]) - }) - }) - - describe('configuration change', () => { - it('marks token-b to removal from slices state', () => { - const expectedSlices: SliceHash[] = ['token-a', 'token-c'] - const actualSlices: SliceState[] = [ - { sliceHash: 'token-a', height: 100 }, - { sliceHash: 'token-b', height: 200 }, - { sliceHash: 'token-c', height: 300 }, - ] - const from = 100 - const to = 200 - - const { toUpdate, toRemove } = diffSlices( - expectedSlices, - actualSlices, - from, - to, - ) - - expect(toRemove).toEqual(['token-b']) - expect(toUpdate).toEqual([{ sliceHash: 'token-a', from, to }]) - }) - - it('marks token-d to addition to slices state', () => { - const expectedSlices: SliceHash[] = [ - 'token-a', - 'token-b', - 'token-c', - 'token-d', - ] - - const actualSlices: SliceState[] = [ - { sliceHash: 'token-a', height: 200 }, - { sliceHash: 'token-b', height: 200 }, - { sliceHash: 'token-c', height: 200 }, - ] - - const from = 100 - const to = 200 - - const { toUpdate, toRemove } = diffSlices( - expectedSlices, - actualSlices, - from, - to, - ) - - expect(toRemove).toEqual([]) - expect(toUpdate).toEqual([{ sliceHash: 'token-d', from: 100, to: 200 }]) - }) - }) -}) - -function testRepositoryFactory() { - const sliceHeights = new Map() - const sliceData = new Map>() - let safeHeight = 0 - - return { - getSliceHeights: mockFn(() => sliceHeights), - getSliceData: mockFn( - (hash: string) => sliceData.get(hash) ?? new Map(), - ), - removeSlices: mockFn((hashes: string[]) => { - for (const hash of hashes) { - sliceHeights.delete(hash) - sliceData.delete(hash) - } - }), - setSliceHeight: mockFn((hash: string, height: number) => { - sliceHeights.set(hash, height) - }), - setSliceData: mockFn((hash: string, data: Map) => { - sliceData.set(hash, data) - }), - getSafeHeight: mockFn(() => { - return safeHeight - }), - setSafeHeight: mockFn((height: number) => { - safeHeight = height - }), - } -} - -class TestSliceIndexer extends SliceIndexer { - public resolveInvalidate: (to: number) => void = () => {} - public rejectInvalidate: (error: unknown) => void = () => {} - - public dispatchCounter = 0 - - async finishInvalidate(result: number | Error): Promise { - await waitUntil(() => this.invalidating) - const counter = this.dispatchCounter - if (typeof result === 'number') { - this.resolveInvalidate(result) - } else { - this.rejectInvalidate(result) - } - await waitUntil(() => this.dispatchCounter > counter) - } - - public invalidating = false - public invalidateTo = 0 - - constructor( - private readonly slices: string[], - private readonly repository: ReturnType, - parents: BaseIndexer[], - name?: string, - retryStrategy?: { - invalidateRetryStrategy?: RetryStrategy - updateRetryStrategy?: RetryStrategy - }, - ) { - super(Logger.SILENT.tag(name), parents, retryStrategy ?? {}) - - const oldDispatch = Reflect.get(this, 'dispatch') - Reflect.set(this, 'dispatch', (action: IndexerAction) => { - oldDispatch.call(this, action) - this.dispatchCounter++ - }) - } - - override getMainSafeHeight(): Promise { - return Promise.resolve(this.repository.getSafeHeight()) - } - - override setMainSafeHeight(safeHeight: number): Promise { - this.repository.setSafeHeight(safeHeight) - return Promise.resolve() - } - - override async invalidate(to: number): Promise { - this.invalidating = true - this.invalidateTo = to - return new Promise((resolve, reject) => { - this.resolveInvalidate = resolve - this.rejectInvalidate = reject - }).finally(() => { - this.invalidating = false - }) - } - - getExpectedSlices(): string[] { - return this.slices - } - - getSliceStates(): Promise { - const sliceHeights = this.repository.getSliceHeights() - const states = [...sliceHeights.entries()].map( - ([slice, height]): SliceState => ({ - sliceHash: slice, - height, - }), - ) - return Promise.resolve(states) - } - - removeSlices(hashes: string[]): Promise { - this.repository.removeSlices(hashes) - return Promise.resolve() - } - - async updateSlices(updates: SliceUpdate[]): Promise { - let minHeight = Infinity - for (const update of updates) { - const sliceData = this.repository.getSliceData(update.sliceHash) - - for (let i = update.from; i <= update.to; i++) { - sliceData.set(i, i * 2) - } - if (update.to < minHeight) { - minHeight = update.to - } - - this.repository.setSliceData(update.sliceHash, sliceData) - this.repository.setSliceHeight(update.sliceHash, update.to) - } - return Promise.resolve(minHeight) - } -} diff --git a/packages/uif/src/indexers/SliceIndexer.ts b/packages/uif/src/indexers/SliceIndexer.ts deleted file mode 100644 index 7ecbc916..00000000 --- a/packages/uif/src/indexers/SliceIndexer.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { ChildIndexer } from './ChildIndexer' - -export type SliceHash = string - -export interface SliceState { - sliceHash: SliceHash - height: number -} - -export interface SliceUpdate { - sliceHash: SliceHash - from: number - to: number -} - -// TODO: implements IChildIndexer -export abstract class SliceIndexer extends ChildIndexer { - override async update(from: number, to: number): Promise { - const sliceStates = await this.getSliceStates() - const { toUpdate, toRemove } = diffSlices( - this.getExpectedSlices(), - sliceStates, - from, - to, - ) - - this.logger.info('Update', { - amountToUpdate: toUpdate.length, - amountToRemove: toRemove.length, - }) - - if (toRemove.length > 0) { - this.logger.debug('Removing slices', { toRemove }) - await this.removeSlices(toRemove) - this.logger.debug('Removed slices') - } - - if (toUpdate.length > 0) { - this.logger.debug('Updating slices', { toUpdate }) - const newHeight = await this.updateSlices(toUpdate) - this.logger.debug('Updated slices', { newHeight }) - return newHeight - } - - return to - } - - override async initialize(): Promise { - const sliceStates = await this.getSliceStates() - const mainSafeHeight = await this.getMainSafeHeight() - return Math.min(...sliceStates.map((s) => s.height), mainSafeHeight) - } - - override async setSafeHeight(height: number): Promise { - await this.setMainSafeHeight(height) - } - - /** - * @returns Height of the indexer. Should be saved in persistent storage. - */ - abstract getMainSafeHeight(): Promise - - /** - * @param height Height of the indexer. Should be saved in persistent storage. - */ - abstract setMainSafeHeight(height: number): Promise - - /** - * @returns Slices derived from config - */ - abstract getExpectedSlices(): SliceHash[] - - /** - * @returns State for Slices stored in the database - */ - abstract getSliceStates(): Promise - - /** - * @param hashes Slices that are no longer part of config - */ - abstract removeSlices(hashes: SliceHash[]): Promise - - /** - * - * @param slices Slices that are part of config and need to be updated - * @returns Minimum of the heights of updated slices - */ - abstract updateSlices(slices: SliceUpdate[]): Promise -} - -/** - * slices toRemove are slices that are no longer part of config - */ -export function diffSlices( - expectedSlices: SliceHash[], - actualSlices: SliceState[], - from: number, - to: number, -): { toUpdate: SliceUpdate[]; toRemove: SliceHash[] } { - const toUpdate: SliceUpdate[] = [] - const toRemove: SliceHash[] = [] - - for (const slice of actualSlices) { - if (!expectedSlices.includes(slice.sliceHash)) { - toRemove.push(slice.sliceHash) - } else if (slice.height < to) { - toUpdate.push({ - sliceHash: slice.sliceHash, - from: Math.max(slice.height, from), - to: to, - }) - } - } - - for (const slice of expectedSlices) { - if (!actualSlices.find((s) => s.sliceHash === slice)) { - toUpdate.push({ sliceHash: slice, from: from, to: to }) - } - } - - return { toUpdate, toRemove } -} diff --git a/packages/uif/src/indexers/multi/MultiIndexer.test.ts b/packages/uif/src/indexers/multi/MultiIndexer.test.ts index e56238c6..743b83d6 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.test.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.test.ts @@ -1,7 +1,7 @@ import { Logger } from '@l2beat/backend-tools' import { expect, mockFn } from 'earl' -import { IMultiIndexer, MultiIndexer } from './MultiIndexer' +import { MultiIndexer } from './MultiIndexer' import { Configuration, SavedConfiguration } from './types' describe(MultiIndexer.name, () => { @@ -297,10 +297,7 @@ describe(MultiIndexer.name, () => { }) }) -class TestMultiIndexer - extends MultiIndexer - implements IMultiIndexer -{ +class TestMultiIndexer extends MultiIndexer { constructor( configurations: Configuration[], private readonly _saved: SavedConfiguration[], diff --git a/packages/uif/src/indexers/multi/MultiIndexer.ts b/packages/uif/src/indexers/multi/MultiIndexer.ts index 3e094ce9..2667b709 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.ts @@ -3,7 +3,7 @@ import { Logger } from '@l2beat/backend-tools' import { BaseIndexer } from '../../BaseIndexer' import { Height } from '../../height' import { RetryStrategy } from '../../Retries' -import { ChildIndexer, IChildIndexer } from '../ChildIndexer' +import { ChildIndexer } from '../ChildIndexer' import { diffConfigurations } from './diffConfigurations' import { toRanges } from './toRanges' import { @@ -14,7 +14,23 @@ import { UpdateConfiguration, } from './types' -export interface IMultiIndexer { +export abstract class MultiIndexer extends ChildIndexer { + private readonly ranges: ConfigurationRange[] + private saved: SavedConfiguration[] = [] + + constructor( + logger: Logger, + parents: BaseIndexer[], + readonly configurations: Configuration[], + opts?: { + updateRetryStrategy?: RetryStrategy + invalidateRetryStrategy?: RetryStrategy + }, + ) { + super(logger, parents, opts) + this.ranges = toRanges(configurations) + } + /** * Initializes the indexer. It returns the configurations that were saved * previously. In case no configurations were saved, it should return an empty @@ -24,18 +40,7 @@ export interface IMultiIndexer { * previously with `setStoredConfigurations`. It shouldn't call * `setStoredConfigurations` itself. */ - multiInitialize: () => Promise - - /** - * Removes data that was previously synced but because configurations changed - * is no longer valid. The data should be removed for the ranges specified - * in each configuration. It is possible for multiple ranges to share a - * configuration id! - * - * This method can only be called during the initialization of the indexer, - * after `multiInitialize` returns. - */ - removeData: (configurations: RemovalConfiguration[]) => Promise + abstract multiInitialize(): Promise /** * Implements the main data fetching process. It is up to the indexer to @@ -63,11 +68,22 @@ export interface IMultiIndexer { * to that height. Returning a value less than `currentHeight` or greater than * `targetHeight` is not permitted. */ - multiUpdate: ( + abstract multiUpdate( currentHeight: number, targetHeight: number, configurations: UpdateConfiguration[], - ) => Promise + ): Promise + + /** + * Removes data that was previously synced but because configurations changed + * is no longer valid. The data should be removed for the ranges specified + * in each configuration. It is possible for multiple ranges to share a + * configuration id! + * + * This method can only be called during the initialization of the indexer, + * after `multiInitialize` returns. + */ + abstract removeData(configurations: RemovalConfiguration[]): Promise /** * Saves configurations that the indexer should use to sync data. The @@ -77,36 +93,6 @@ export interface IMultiIndexer { * indexer should save the returned configurations and ensure that no other * configurations are persisted. */ - saveConfigurations: (configurations: SavedConfiguration[]) => Promise -} - -export abstract class MultiIndexer - extends ChildIndexer - implements IChildIndexer, IMultiIndexer -{ - private readonly ranges: ConfigurationRange[] - private saved: SavedConfiguration[] = [] - - constructor( - logger: Logger, - parents: BaseIndexer[], - readonly configurations: Configuration[], - opts?: { - updateRetryStrategy?: RetryStrategy - invalidateRetryStrategy?: RetryStrategy - }, - ) { - super(logger, parents, opts) - this.ranges = toRanges(configurations) - } - - abstract multiInitialize(): Promise - abstract multiUpdate( - currentHeight: number, - targetHeight: number, - configurations: UpdateConfiguration[], - ): Promise - abstract removeData(configurations: RemovalConfiguration[]): Promise abstract saveConfigurations( configurations: SavedConfiguration[], ): Promise From 4ace7079788cdf9f2e8c8adb3085138f9959dc84 Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Thu, 21 Mar 2024 23:08:55 +0100 Subject: [PATCH 15/31] Remove old example code --- packages/example/.mocharc.json | 7 - packages/example/src/Application.ts | 125 ------------------ packages/example/src/Config.ts | 17 --- packages/example/src/index.test.ts | 7 - packages/example/src/indexers/ABC_Indexer.ts | 65 --------- .../example/src/indexers/AB_BC_Indexer.ts | 79 ----------- .../example/src/indexers/BalanceIndexer.ts | 39 ------ .../src/indexers/BlockNumberIndexer.ts | 40 ------ .../example/src/indexers/FakeClockIndexer.ts | 20 --- packages/example/src/indexers/TvlIndexer.ts | 40 ------ .../src/repositories/ABC_Repository.ts | 46 ------- .../src/repositories/AB_BC_Repository.ts | 38 ------ .../src/repositories/BalanceRepository.ts | 12 -- .../src/repositories/BlockNumberRepository.ts | 12 -- .../example/src/repositories/TvlRepository.ts | 12 -- packages/{example => uif-example}/.eslintrc | 0 .../{example => uif-example}/.prettierignore | 0 packages/{example => uif-example}/.prettierrc | 0 .../{example => uif-example}/package.json | 2 +- packages/uif-example/src/Application.ts | 21 +++ .../src}/HourlyIndexer.ts | 4 +- .../{example => uif-example}/src/index.ts | 4 +- .../{example => uif-example}/tsconfig.json | 0 23 files changed, 25 insertions(+), 565 deletions(-) delete mode 100644 packages/example/.mocharc.json delete mode 100644 packages/example/src/Application.ts delete mode 100644 packages/example/src/Config.ts delete mode 100644 packages/example/src/index.test.ts delete mode 100644 packages/example/src/indexers/ABC_Indexer.ts delete mode 100644 packages/example/src/indexers/AB_BC_Indexer.ts delete mode 100644 packages/example/src/indexers/BalanceIndexer.ts delete mode 100644 packages/example/src/indexers/BlockNumberIndexer.ts delete mode 100644 packages/example/src/indexers/FakeClockIndexer.ts delete mode 100644 packages/example/src/indexers/TvlIndexer.ts delete mode 100644 packages/example/src/repositories/ABC_Repository.ts delete mode 100644 packages/example/src/repositories/AB_BC_Repository.ts delete mode 100644 packages/example/src/repositories/BalanceRepository.ts delete mode 100644 packages/example/src/repositories/BlockNumberRepository.ts delete mode 100644 packages/example/src/repositories/TvlRepository.ts rename packages/{example => uif-example}/.eslintrc (100%) rename packages/{example => uif-example}/.prettierignore (100%) rename packages/{example => uif-example}/.prettierrc (100%) rename packages/{example => uif-example}/package.json (94%) create mode 100644 packages/uif-example/src/Application.ts rename packages/{example/src/indexers => uif-example/src}/HourlyIndexer.ts (82%) rename packages/{example => uif-example}/src/index.ts (61%) rename packages/{example => uif-example}/tsconfig.json (100%) diff --git a/packages/example/.mocharc.json b/packages/example/.mocharc.json deleted file mode 100644 index 862b85f9..00000000 --- a/packages/example/.mocharc.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "spec": "src/**/*.test.ts", - "require": ["esbuild-register"], - "watchExtensions": "ts", - "extension": "ts", - "reporterOption": "maxDiffSize=0" -} diff --git a/packages/example/src/Application.ts b/packages/example/src/Application.ts deleted file mode 100644 index f238e7ff..00000000 --- a/packages/example/src/Application.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { Logger } from '@l2beat/backend-tools' -import { BaseIndexer, Retries } from '@l2beat/uif' - -import { Config } from './Config' -import { AB_BC_Indexer } from './indexers/AB_BC_Indexer' -import { ABC_Indexer } from './indexers/ABC_Indexer' -import { BalanceIndexer } from './indexers/BalanceIndexer' -import { BlockNumberIndexer } from './indexers/BlockNumberIndexer' -import { FakeClockIndexer } from './indexers/FakeClockIndexer' -import { TvlIndexer } from './indexers/TvlIndexer' -import { AB_BC_Repository } from './repositories/AB_BC_Repository' -import { ABC_Repository } from './repositories/ABC_Repository' -import { BalanceRepository } from './repositories/BalanceRepository' -import { BlockNumberRepository } from './repositories/BlockNumberRepository' -import { TvlRepository } from './repositories/TvlRepository' - -interface Module { - start: () => Promise -} - -export class Application { - start: () => Promise - - constructor(config: Config) { - const logger = new Logger({ - logLevel: 'DEBUG', - format: 'pretty', - colors: true, - utc: true, - }) - - const modules = [ - createMainModule(config, logger), - createABCModule(config, logger), - ] - - this.start = async (): Promise => { - console.log(`Application started: ${config.name}`) - await Promise.all(modules.map((module) => module?.start())) - } - } -} - -function createABCModule(config: Config, logger: Logger): Module | undefined { - if (!config.modules.abc) { - return undefined - } - - const blockNumberRepository = new BlockNumberRepository() - const abc_repository = new ABC_Repository() - const ab_bc_repository = new AB_BC_Repository() - - const fakeClockIndexer = new FakeClockIndexer( - logger.configure({ logLevel: 'NONE' }), - ) - const blockNumberIndexer = new BlockNumberIndexer( - logger.configure({ logLevel: 'NONE' }), - fakeClockIndexer, - blockNumberRepository, - ) - const abc_Indexer = new ABC_Indexer( - logger, - blockNumberIndexer, - abc_repository, - ) - const ab_bc_indexer = new AB_BC_Indexer( - logger, - abc_Indexer, - ab_bc_repository, - abc_repository, - ) - - return { - start: async (): Promise => { - console.log(`Module started: ABCModule`) - - await fakeClockIndexer.start() - await blockNumberIndexer.start() - await abc_Indexer.start() - await ab_bc_indexer.start() - }, - } -} - -function createMainModule(config: Config, logger: Logger): Module | undefined { - if (!config.modules.main) { - return undefined - } - - const blockNumberRepository = new BlockNumberRepository() - const balanceRepository = new BalanceRepository() - const tvlRepository = new TvlRepository() - - BaseIndexer.GET_DEFAULT_RETRY_STRATEGY = () => - Retries.exponentialBackOff({ - initialTimeoutMs: 100, - maxAttempts: 10, - maxTimeoutMs: 60 * 1000, - }) - - const fakeClockIndexer = new FakeClockIndexer(logger) - const blockNumberIndexer = new BlockNumberIndexer( - logger, - fakeClockIndexer, - blockNumberRepository, - ) - const balanceIndexer = new BalanceIndexer( - logger, - blockNumberIndexer, - balanceRepository, - ) - const tvlIndexer = new TvlIndexer(logger, balanceIndexer, tvlRepository) - - return { - start: async (): Promise => { - await Promise.resolve() - console.log(`Application started: MainModule`) - - await fakeClockIndexer.start() - await blockNumberIndexer.start() - await balanceIndexer.start() - await tvlIndexer.start() - }, - } -} diff --git a/packages/example/src/Config.ts b/packages/example/src/Config.ts deleted file mode 100644 index cfbbd76e..00000000 --- a/packages/example/src/Config.ts +++ /dev/null @@ -1,17 +0,0 @@ -export interface Config { - name: string - modules: { - main: boolean - abc: boolean - } -} - -export function getConfig(): Config { - return { - name: 'uif-example', - modules: { - main: false, - abc: true, - }, - } -} diff --git a/packages/example/src/index.test.ts b/packages/example/src/index.test.ts deleted file mode 100644 index 7736fd3e..00000000 --- a/packages/example/src/index.test.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { expect } from 'earl' - -describe('plus operator', () => { - it('adds two numbers', () => { - expect(1 + 2).toEqual(3) - }) -}) diff --git a/packages/example/src/indexers/ABC_Indexer.ts b/packages/example/src/indexers/ABC_Indexer.ts deleted file mode 100644 index 3c5b1bae..00000000 --- a/packages/example/src/indexers/ABC_Indexer.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { Logger } from '@l2beat/backend-tools' -import { SliceIndexer, SliceState, SliceUpdate } from '@l2beat/uif' - -import { ABC_Repository } from '../repositories/ABC_Repository' -import { BlockNumberIndexer } from './BlockNumberIndexer' - -export class ABC_Indexer extends SliceIndexer { - constructor( - logger: Logger, - blockNumberIndexer: BlockNumberIndexer, - private readonly abc_repository: ABC_Repository, - private readonly tokens = ['A', 'B', 'C'], - ) { - super(logger, [blockNumberIndexer]) - } - - override getExpectedSlices(): string[] { - return this.tokens - } - - override async getSliceStates(): Promise { - const tokenHeights = await this.abc_repository.getSliceHeights() - const states = [...tokenHeights.entries()].map( - ([token, height]): SliceState => ({ - sliceHash: token, - height, - }), - ) - return states - } - - override async removeSlices(hashes: string[]): Promise { - return await this.abc_repository.removeSlices(hashes) - } - - override async updateSlices(updates: SliceUpdate[]): Promise { - let minHeight = Infinity - for (const update of updates) { - const balances = await this.abc_repository.getSliceData(update.sliceHash) - - for (let i = update.from; i <= update.to; i++) { - balances.set(i, i * 2) - } - if (update.to < minHeight) { - minHeight = update.to - } - - await this.abc_repository.setSliceData(update.sliceHash, balances) - await this.abc_repository.setSliceHeight(update.sliceHash, update.to) - } - return minHeight - } - - override async getMainSafeHeight(): Promise { - return await this.abc_repository.getSafeHeight() - } - - override async setMainSafeHeight(height: number): Promise { - await this.abc_repository.setSafeHeight(height) - } - - override invalidate(targetHeight: number): Promise { - return Promise.resolve(targetHeight) - } -} diff --git a/packages/example/src/indexers/AB_BC_Indexer.ts b/packages/example/src/indexers/AB_BC_Indexer.ts deleted file mode 100644 index 3aa3c997..00000000 --- a/packages/example/src/indexers/AB_BC_Indexer.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { Logger } from '@l2beat/backend-tools' -import { SliceIndexer, SliceState, SliceUpdate } from '@l2beat/uif' - -import { AB_BC_Repository } from '../repositories/AB_BC_Repository' -import { ABC_Repository } from '../repositories/ABC_Repository' -import { ABC_Indexer } from './ABC_Indexer' - -export class AB_BC_Indexer extends SliceIndexer { - private readonly slices = new Map() - - constructor( - logger: Logger, - abc_indexer: ABC_Indexer, - private readonly ab_bc_repository: AB_BC_Repository, - private readonly abc_repository: ABC_Repository, - private readonly sums = [ - ['A', 'B'], - ['B', 'C'], - ], - ) { - super(logger, [abc_indexer]) - this.slices = new Map( - this.sums.map((sum) => [sum.join('+'), sum]), - ) - } - - override getExpectedSlices(): string[] { - return [...this.slices.keys()] - } - - override async getSliceStates(): Promise { - const sliceHeights = await this.ab_bc_repository.getSliceHeights() - const states = [...sliceHeights.entries()].map( - ([sliceHash, height]): SliceState => ({ sliceHash, height }), - ) - return Promise.resolve(states) - } - - override async removeSlices(hashes: string[]): Promise { - await this.ab_bc_repository.removeSlices(hashes) - } - - override async updateSlices(updates: SliceUpdate[]): Promise { - let minHeight = Infinity - for (const update of updates) { - const sumMap = await this.ab_bc_repository.getSliceData(update.sliceHash) - - for (let i = update.from; i <= update.to; i++) { - const tokens = this.slices.get(update.sliceHash) ?? [] - const values = await Promise.all( - tokens.map((token) => this.abc_repository.getTokenBalance(token, i)), - ) - sumMap.set( - i, - values.reduce((a, b) => a + b, 0), - ) - } - if (update.to < minHeight) { - minHeight = update.to - } - - await this.ab_bc_repository.setSliceData(update.sliceHash, sumMap) - await this.ab_bc_repository.setSliceHeight(update.sliceHash, update.to) - } - return Promise.resolve(minHeight) - } - - override async getMainSafeHeight(): Promise { - return await this.ab_bc_repository.getSafeHeight() - } - - override async setMainSafeHeight(height: number): Promise { - await this.ab_bc_repository.setSafeHeight(height) - } - - override async invalidate(targetHeight: number): Promise { - return Promise.resolve(targetHeight) - } -} diff --git a/packages/example/src/indexers/BalanceIndexer.ts b/packages/example/src/indexers/BalanceIndexer.ts deleted file mode 100644 index c919b237..00000000 --- a/packages/example/src/indexers/BalanceIndexer.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { Logger } from '@l2beat/backend-tools' -import { ChildIndexer } from '@l2beat/uif' -import { setTimeout } from 'timers/promises' - -import { BalanceRepository } from '../repositories/BalanceRepository' -import { BlockNumberIndexer } from './BlockNumberIndexer' - -export class BalanceIndexer extends ChildIndexer { - constructor( - logger: Logger, - blockNumberIndexer: BlockNumberIndexer, - private readonly balanceRepository: BalanceRepository, - ) { - super(logger, [blockNumberIndexer]) - } - - override async update(from: number, to: number): Promise { - await setTimeout(1_000) - if (Math.random() < 0.5) { - this.logger.info('BalanceIndexer: height decreased') - return Math.max(from - 500, 0) - } - to = Math.min(from + 100, to) - return to - } - - override async invalidate(targetHeight: number): Promise { - return Promise.resolve(targetHeight) - } - - override async getSafeHeight(): Promise { - const height = await this.balanceRepository.getLastSynced() - return height ?? 0 - } - - override async setSafeHeight(height: number): Promise { - return this.balanceRepository.setLastSynced(height) - } -} diff --git a/packages/example/src/indexers/BlockNumberIndexer.ts b/packages/example/src/indexers/BlockNumberIndexer.ts deleted file mode 100644 index a4c239f6..00000000 --- a/packages/example/src/indexers/BlockNumberIndexer.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { Logger } from '@l2beat/backend-tools' -import { ChildIndexer } from '@l2beat/uif' -import { setTimeout } from 'timers/promises' - -import { BlockNumberRepository } from '../repositories/BlockNumberRepository' -import { FakeClockIndexer } from './FakeClockIndexer' - -export class BlockNumberIndexer extends ChildIndexer { - constructor( - logger: Logger, - fakeClockIndexer: FakeClockIndexer, - private readonly blockNumberRepository: BlockNumberRepository, - ) { - super(logger, [fakeClockIndexer]) - } - - override async update(from: number, targetHeight: number): Promise { - await setTimeout(2_000) - if (Math.random() < 0.5) { - throw new Error('Random error while updating') - } - return targetHeight - } - - override async invalidate(targetHeight: number): Promise { - if (Math.random() < 0.5) { - throw new Error('Random error while invalidating') - } - return Promise.resolve(targetHeight) - } - - override async getSafeHeight(): Promise { - const height = await this.blockNumberRepository.getLastSynced() - return height ?? 0 - } - - override async setSafeHeight(height: number): Promise { - return this.blockNumberRepository.setLastSynced(height) - } -} diff --git a/packages/example/src/indexers/FakeClockIndexer.ts b/packages/example/src/indexers/FakeClockIndexer.ts deleted file mode 100644 index c6090b66..00000000 --- a/packages/example/src/indexers/FakeClockIndexer.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { RootIndexer } from '@l2beat/uif' - -export class FakeClockIndexer extends RootIndexer { - private height = 100 - - override async start(): Promise { - await super.start() - setInterval(() => this.requestTick(), 1_000) - } - - override async tick(): Promise { - if (Math.random() < 0.05) { - this.height = Math.max(this.height - 50, 0) - this.logger.info('FakeClockIndexer: height decreased') - } else { - this.height += 10 - } - return Promise.resolve(this.height) - } -} diff --git a/packages/example/src/indexers/TvlIndexer.ts b/packages/example/src/indexers/TvlIndexer.ts deleted file mode 100644 index 9406f392..00000000 --- a/packages/example/src/indexers/TvlIndexer.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { Logger } from '@l2beat/backend-tools' -import { ChildIndexer } from '@l2beat/uif' -import { setTimeout } from 'timers/promises' - -import { TvlRepository } from '../repositories/TvlRepository' -import { BalanceIndexer } from './BalanceIndexer' - -export class TvlIndexer extends ChildIndexer { - height = 0 - - constructor( - logger: Logger, - balanceIndexer: BalanceIndexer, - private readonly tvlRepository: TvlRepository, - ) { - super(logger, [balanceIndexer]) - } - - override async update(from: number, to: number): Promise { - await setTimeout(500) - to = Math.min(from + 10, to) - this.height = to - return to - } - - override async invalidate(targetHeight: number): Promise { - const newHeight = Math.max(this.height - 5, targetHeight) - this.height = newHeight - return await Promise.resolve(newHeight) - } - - override async getSafeHeight(): Promise { - const height = await this.tvlRepository.getLastSynced() - return height ?? 0 - } - - override async setSafeHeight(height: number): Promise { - await this.tvlRepository.setLastSynced(height) - } -} diff --git a/packages/example/src/repositories/ABC_Repository.ts b/packages/example/src/repositories/ABC_Repository.ts deleted file mode 100644 index e3c22cd2..00000000 --- a/packages/example/src/repositories/ABC_Repository.ts +++ /dev/null @@ -1,46 +0,0 @@ -type SliceHeights = Map -type SliceData = Map - -export class ABC_Repository { - private readonly sliceHeights = new Map() - private readonly slicesData = new Map>() - private safeHeight = 0 - - getSliceHeights(): Promise { - return Promise.resolve(this.sliceHeights) - } - getSliceData(hash: string): Promise { - const sliceData = this.slicesData.get(hash) ?? new Map() - return Promise.resolve(sliceData) - } - async removeSlices(hashes: string[]): Promise { - await Promise.resolve() - for (const hash of hashes) { - this.sliceHeights.delete(hash) - this.slicesData.delete(hash) - } - } - async setSliceHeight(hash: string, height: number): Promise { - await Promise.resolve() - this.sliceHeights.set(hash, height) - } - async setSliceData(hash: string, data: Map): Promise { - await Promise.resolve() - this.slicesData.set(hash, data) - } - getSafeHeight(): Promise { - return Promise.resolve(this.safeHeight) - } - async setSafeHeight(height: number): Promise { - await Promise.resolve() - this.safeHeight = height - } - - getTokenBalance(token: string, blockNumber: number): Promise { - const balance = this.slicesData.get(token)?.get(blockNumber) - if (balance === undefined) { - throw new Error(`No balance of ${token} for ${blockNumber}`) - } - return Promise.resolve(balance) - } -} diff --git a/packages/example/src/repositories/AB_BC_Repository.ts b/packages/example/src/repositories/AB_BC_Repository.ts deleted file mode 100644 index dd44d217..00000000 --- a/packages/example/src/repositories/AB_BC_Repository.ts +++ /dev/null @@ -1,38 +0,0 @@ -type SliceHeights = Map -type SliceData = Map - -export class AB_BC_Repository { - private readonly sliceHeights = new Map() - private readonly slicesData = new Map>() - private safeHeight = 0 - - getSliceHeights(): Promise { - return Promise.resolve(this.sliceHeights) - } - getSliceData(hash: string): Promise { - const sliceData = this.slicesData.get(hash) ?? new Map() - return Promise.resolve(sliceData) - } - async removeSlices(hashes: string[]): Promise { - await Promise.resolve() - for (const hash of hashes) { - this.sliceHeights.delete(hash) - this.slicesData.delete(hash) - } - } - async setSliceHeight(hash: string, height: number): Promise { - await Promise.resolve() - this.sliceHeights.set(hash, height) - } - async setSliceData(hash: string, data: Map): Promise { - await Promise.resolve() - this.slicesData.set(hash, data) - } - getSafeHeight(): Promise { - return Promise.resolve(this.safeHeight) - } - async setSafeHeight(height: number): Promise { - await Promise.resolve() - this.safeHeight = height - } -} diff --git a/packages/example/src/repositories/BalanceRepository.ts b/packages/example/src/repositories/BalanceRepository.ts deleted file mode 100644 index 0e15caee..00000000 --- a/packages/example/src/repositories/BalanceRepository.ts +++ /dev/null @@ -1,12 +0,0 @@ -export class BalanceRepository { - private lastSynced: number | undefined = undefined - - async getLastSynced(): Promise { - return Promise.resolve(this.lastSynced) - } - - async setLastSynced(lastSynced: number): Promise { - this.lastSynced = lastSynced - return Promise.resolve() - } -} diff --git a/packages/example/src/repositories/BlockNumberRepository.ts b/packages/example/src/repositories/BlockNumberRepository.ts deleted file mode 100644 index c1a20bb0..00000000 --- a/packages/example/src/repositories/BlockNumberRepository.ts +++ /dev/null @@ -1,12 +0,0 @@ -export class BlockNumberRepository { - private lastSynced: number | undefined = undefined - - async getLastSynced(): Promise { - return Promise.resolve(this.lastSynced) - } - - async setLastSynced(lastSynced: number): Promise { - this.lastSynced = lastSynced - return Promise.resolve() - } -} diff --git a/packages/example/src/repositories/TvlRepository.ts b/packages/example/src/repositories/TvlRepository.ts deleted file mode 100644 index 4d85b6b6..00000000 --- a/packages/example/src/repositories/TvlRepository.ts +++ /dev/null @@ -1,12 +0,0 @@ -export class TvlRepository { - private lastSynced: number | undefined = undefined - - async getLastSynced(): Promise { - return Promise.resolve(this.lastSynced) - } - - async setLastSynced(lastSynced: number): Promise { - this.lastSynced = lastSynced - return Promise.resolve() - } -} diff --git a/packages/example/.eslintrc b/packages/uif-example/.eslintrc similarity index 100% rename from packages/example/.eslintrc rename to packages/uif-example/.eslintrc diff --git a/packages/example/.prettierignore b/packages/uif-example/.prettierignore similarity index 100% rename from packages/example/.prettierignore rename to packages/uif-example/.prettierignore diff --git a/packages/example/.prettierrc b/packages/uif-example/.prettierrc similarity index 100% rename from packages/example/.prettierrc rename to packages/uif-example/.prettierrc diff --git a/packages/example/package.json b/packages/uif-example/package.json similarity index 94% rename from packages/example/package.json rename to packages/uif-example/package.json index 6ecd57d0..4bd9399c 100644 --- a/packages/example/package.json +++ b/packages/uif-example/package.json @@ -1,5 +1,5 @@ { - "name": "example", + "name": "uif-example", "license": "MIT", "version": "0.1.0", "private": true, diff --git a/packages/uif-example/src/Application.ts b/packages/uif-example/src/Application.ts new file mode 100644 index 00000000..f7462b73 --- /dev/null +++ b/packages/uif-example/src/Application.ts @@ -0,0 +1,21 @@ +import { Logger } from '@l2beat/backend-tools' + +import { HourlyIndexer } from './HourlyIndexer' + +export class Application { + start: () => Promise + + constructor() { + const logger = Logger.DEBUG + + const hourlyIndexer = new HourlyIndexer(logger) + + this.start = async (): Promise => { + logger.for('Application').info('Starting') + + hourlyIndexer.start() + + logger.for('Application').info('Started') + } + } +} diff --git a/packages/example/src/indexers/HourlyIndexer.ts b/packages/uif-example/src/HourlyIndexer.ts similarity index 82% rename from packages/example/src/indexers/HourlyIndexer.ts rename to packages/uif-example/src/HourlyIndexer.ts index 2fa9bef2..34eb6436 100644 --- a/packages/example/src/indexers/HourlyIndexer.ts +++ b/packages/uif-example/src/HourlyIndexer.ts @@ -1,9 +1,9 @@ import { RootIndexer } from '@l2beat/uif' export class HourlyIndexer extends RootIndexer { - override async start(): Promise { - await super.start() + async initialize(): Promise { setInterval(() => this.requestTick(), 60 * 1000) + return this.tick() } async tick(): Promise { diff --git a/packages/example/src/index.ts b/packages/uif-example/src/index.ts similarity index 61% rename from packages/example/src/index.ts rename to packages/uif-example/src/index.ts index 5e062fb9..26f75c5f 100644 --- a/packages/example/src/index.ts +++ b/packages/uif-example/src/index.ts @@ -1,5 +1,4 @@ import { Application } from './Application' -import { getConfig } from './Config' main().catch((e) => { console.error(e) @@ -7,7 +6,6 @@ main().catch((e) => { }) async function main(): Promise { - const config = getConfig() - const application = new Application(config) + const application = new Application() await application.start() } diff --git a/packages/example/tsconfig.json b/packages/uif-example/tsconfig.json similarity index 100% rename from packages/example/tsconfig.json rename to packages/uif-example/tsconfig.json From 9b21f1c73c44b0a2834095c08e5adc5bb5abad37 Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Thu, 21 Mar 2024 23:45:07 +0100 Subject: [PATCH 16/31] Add example of a PriceIndexer --- packages/uif-example/src/Application.ts | 52 +++++++++- .../uif-example/src/prices/PriceConfig.ts | 4 + .../uif-example/src/prices/PriceIndexer.ts | 99 +++++++++++++++++++ .../src/prices/PriceIndexerRepository.ts | 19 ++++ .../uif-example/src/prices/PriceRepository.ts | 19 ++++ .../uif-example/src/prices/PriceService.ts | 17 ++++ .../{BaseIndexer.test.ts => Indexer.test.ts} | 6 +- .../uif/src/{BaseIndexer.ts => Indexer.ts} | 30 +++--- packages/uif/src/index.ts | 4 +- packages/uif/src/indexers/ChildIndexer.ts | 4 +- packages/uif/src/indexers/RootIndexer.ts | 7 +- .../src/indexers/multi/MultiIndexer.test.ts | 8 +- .../uif/src/indexers/multi/MultiIndexer.ts | 29 +++--- .../indexers/multi/diffConfigurations.test.ts | 4 +- .../src/indexers/multi/diffConfigurations.ts | 21 ++-- packages/uif/src/indexers/multi/types.ts | 6 +- 16 files changed, 271 insertions(+), 58 deletions(-) create mode 100644 packages/uif-example/src/prices/PriceConfig.ts create mode 100644 packages/uif-example/src/prices/PriceIndexer.ts create mode 100644 packages/uif-example/src/prices/PriceIndexerRepository.ts create mode 100644 packages/uif-example/src/prices/PriceRepository.ts create mode 100644 packages/uif-example/src/prices/PriceService.ts rename packages/uif/src/{BaseIndexer.test.ts => Indexer.test.ts} (99%) rename packages/uif/src/{BaseIndexer.ts => Indexer.ts} (94%) diff --git a/packages/uif-example/src/Application.ts b/packages/uif-example/src/Application.ts index f7462b73..cb7aa8e7 100644 --- a/packages/uif-example/src/Application.ts +++ b/packages/uif-example/src/Application.ts @@ -1,6 +1,10 @@ import { Logger } from '@l2beat/backend-tools' import { HourlyIndexer } from './HourlyIndexer' +import { PriceIndexer } from './prices/PriceIndexer' +import { PriceIndexerRepository } from './prices/PriceIndexerRepository' +import { PriceRepository } from './prices/PriceRepository' +import { PriceService } from './prices/PriceService' export class Application { start: () => Promise @@ -10,10 +14,56 @@ export class Application { const hourlyIndexer = new HourlyIndexer(logger) + const priceService = new PriceService() + const priceRepository = new PriceRepository() + const priceIndexerRepository = new PriceIndexerRepository() + + const ethereumPriceIndexer = new PriceIndexer( + 'price-ethereum', + priceService, + priceRepository, + priceIndexerRepository, + logger, + [hourlyIndexer], + [ + { + // could be a hash of properties & minHeight instead + id: 'eth-ethereum', + properties: { tokenSymbol: 'ETH', apiId: 'ethereum' }, + minHeight: new Date('2021-01-01T00:00:00Z').getTime(), + maxHeight: null, + }, + { + id: 'weth-ethereum', + properties: { tokenSymbol: 'WETH', apiId: 'ethereum' }, + minHeight: new Date('2022-01-01T00:00:00Z').getTime(), + maxHeight: null, + }, + ], + ) + const bitcoinPriceIndexer = new PriceIndexer( + 'price-bitcoin', + priceService, + priceRepository, + priceIndexerRepository, + logger, + [hourlyIndexer], + [ + { + id: 'btc-bitcoin', + properties: { tokenSymbol: 'BTC', apiId: 'bitcoin' }, + minHeight: new Date('2022-01-01T00:00:00Z').getTime(), + maxHeight: null, + }, + ], + ) + this.start = async (): Promise => { logger.for('Application').info('Starting') - hourlyIndexer.start() + await hourlyIndexer.start() + await ethereumPriceIndexer.start() + await bitcoinPriceIndexer.start() logger.for('Application').info('Started') } diff --git a/packages/uif-example/src/prices/PriceConfig.ts b/packages/uif-example/src/prices/PriceConfig.ts new file mode 100644 index 00000000..1dbab632 --- /dev/null +++ b/packages/uif-example/src/prices/PriceConfig.ts @@ -0,0 +1,4 @@ +export interface PriceConfig { + tokenSymbol: string + apiId: string +} diff --git a/packages/uif-example/src/prices/PriceIndexer.ts b/packages/uif-example/src/prices/PriceIndexer.ts new file mode 100644 index 00000000..3e9c42b0 --- /dev/null +++ b/packages/uif-example/src/prices/PriceIndexer.ts @@ -0,0 +1,99 @@ +import { Logger } from '@l2beat/backend-tools' +import { + Configuration, + Indexer, + IndexerOptions, + MultiIndexer, + RemovalConfiguration, + SavedConfiguration, + UpdateConfiguration, +} from '@l2beat/uif' + +import { PriceConfig } from './PriceConfig' +import { PriceIndexerRepository } from './PriceIndexerRepository' +import { PriceRepository } from './PriceRepository' +import { PriceService } from './PriceService' + +const ONE_HOUR = 60 * 60 * 1000 + +export class PriceIndexer extends MultiIndexer { + private readonly apiId: string + + constructor( + private readonly indexerId: string, + private readonly priceService: PriceService, + private readonly priceRepository: PriceRepository, + private readonly priceIndexerRepository: PriceIndexerRepository, + logger: Logger, + parents: Indexer[], + configurations: Configuration[], + options?: IndexerOptions, + ) { + super(logger, parents, configurations, options) + const apiId = configurations[0]?.properties.apiId + if (!apiId) { + throw new Error('At least one configuration is required') + } + if (configurations.some((c) => c.properties.apiId !== apiId)) { + throw new Error('All configurations must have the same apiId') + } + this.apiId = apiId + } + + override async multiInitialize(): Promise[]> { + return this.priceIndexerRepository.load(this.indexerId) + } + + override async multiUpdate( + currentHeight: number, + targetHeight: number, + configurations: UpdateConfiguration[], + ): Promise { + const startHour = currentHeight - (currentHeight % ONE_HOUR) + ONE_HOUR + const endHour = Math.min( + targetHeight - (targetHeight % ONE_HOUR), + // for example the api costs us more money for larger ranges + startHour + 23 * ONE_HOUR, + ) + + if (startHour >= endHour) { + return startHour + } + + const prices = await this.priceService.getHourlyPrices( + this.apiId, + startHour, + endHour, + ) + + const dataToSave = configurations.flatMap((configuration) => { + return prices.map(({ timestamp, price }) => ({ + tokenSymbol: configuration.properties.tokenSymbol, + timestamp, + price, + })) + }) + await this.priceRepository.save(dataToSave) + + // TODO: Maybe if targetHeight is not exactly an hour we can return it? + return endHour + } + + override async removeData( + configurations: RemovalConfiguration[], + ): Promise { + for (const c of configurations) { + await this.priceRepository.deletePrices( + c.properties.tokenSymbol, + c.fromHeightInclusive, + c.toHeightInclusive, + ) + } + } + + override async saveConfigurations( + configurations: SavedConfiguration[], + ): Promise { + return this.priceIndexerRepository.save(this.indexerId, configurations) + } +} diff --git a/packages/uif-example/src/prices/PriceIndexerRepository.ts b/packages/uif-example/src/prices/PriceIndexerRepository.ts new file mode 100644 index 00000000..6fcac7fd --- /dev/null +++ b/packages/uif-example/src/prices/PriceIndexerRepository.ts @@ -0,0 +1,19 @@ +import { SavedConfiguration } from '@l2beat/uif' + +import { PriceConfig } from './PriceConfig' + +export class PriceIndexerRepository { + private data: Record[]> = {} + + async save( + indexerId: string, + configurations: SavedConfiguration[], + ): Promise { + this.data[indexerId] = configurations + return Promise.resolve() + } + + async load(indexerId: string): Promise[]> { + return Promise.resolve(this.data[indexerId] ?? []) + } +} diff --git a/packages/uif-example/src/prices/PriceRepository.ts b/packages/uif-example/src/prices/PriceRepository.ts new file mode 100644 index 00000000..aafa9d71 --- /dev/null +++ b/packages/uif-example/src/prices/PriceRepository.ts @@ -0,0 +1,19 @@ +export class PriceRepository { + async save( + prices: { tokenSymbol: string; timestamp: number; price: number }[], + ): Promise { + prices // use it so that eslint doesn't complain + return Promise.resolve() + } + + async deletePrices( + tokenSymbol: string, + fromTimestampInclusive: number, + toTimestampInclusive: number, + ): Promise { + tokenSymbol // use it so that eslint doesn't complain + fromTimestampInclusive // use it so that eslint doesn't complain + toTimestampInclusive // use it so that eslint doesn't complain + return Promise.resolve() + } +} diff --git a/packages/uif-example/src/prices/PriceService.ts b/packages/uif-example/src/prices/PriceService.ts new file mode 100644 index 00000000..759ec11f --- /dev/null +++ b/packages/uif-example/src/prices/PriceService.ts @@ -0,0 +1,17 @@ +const ONE_HOUR = 60 * 60 * 1000 + +export class PriceService { + async getHourlyPrices( + apiId: string, + startHourInclusive: number, + endHourInclusive: number, + ): Promise<{ timestamp: number; price: number }[]> { + apiId // use it so that eslint doesn't complain + + const prices: { timestamp: number; price: number }[] = [] + for (let t = startHourInclusive; t <= endHourInclusive; t += ONE_HOUR) { + prices.push({ timestamp: t, price: Math.random() * 1000 }) + } + return Promise.resolve(prices) + } +} diff --git a/packages/uif/src/BaseIndexer.test.ts b/packages/uif/src/Indexer.test.ts similarity index 99% rename from packages/uif/src/BaseIndexer.test.ts rename to packages/uif/src/Indexer.test.ts index 2199f7a6..49c7419e 100644 --- a/packages/uif/src/BaseIndexer.test.ts +++ b/packages/uif/src/Indexer.test.ts @@ -2,13 +2,13 @@ import { Logger } from '@l2beat/backend-tools' import { install } from '@sinonjs/fake-timers' import { expect, mockFn } from 'earl' -import { BaseIndexer } from './BaseIndexer' +import { Indexer } from './Indexer' import { ChildIndexer } from './indexers/ChildIndexer' import { RootIndexer } from './indexers/RootIndexer' import { IndexerAction } from './reducer/types/IndexerAction' import { RetryStrategy } from './Retries' -describe(BaseIndexer.name, () => { +describe(Indexer.name, () => { describe('correctly informs about updates', () => { it('first invalidate then parent update', async () => { const parent = new TestRootIndexer(0) @@ -324,7 +324,7 @@ class TestChildIndexer extends ChildIndexer { public invalidateTo = 0 constructor( - parents: BaseIndexer[], + parents: Indexer[], private testSafeHeight: number, name?: string, retryStrategy?: { diff --git a/packages/uif/src/BaseIndexer.ts b/packages/uif/src/Indexer.ts similarity index 94% rename from packages/uif/src/BaseIndexer.ts rename to packages/uif/src/Indexer.ts index e6234f6e..0a7a8e70 100644 --- a/packages/uif/src/BaseIndexer.ts +++ b/packages/uif/src/Indexer.ts @@ -16,8 +16,14 @@ import { import { IndexerState } from './reducer/types/IndexerState' import { Retries, RetryStrategy } from './Retries' -export abstract class BaseIndexer { - private readonly children: BaseIndexer[] = [] +export interface IndexerOptions { + tickRetryStrategy?: RetryStrategy + updateRetryStrategy?: RetryStrategy + invalidateRetryStrategy?: RetryStrategy +} + +export abstract class Indexer { + private readonly children: Indexer[] = [] /** * This can be overridden to provide a custom retry strategy. It will be @@ -123,12 +129,8 @@ export abstract class BaseIndexer { constructor( protected logger: Logger, - public readonly parents: BaseIndexer[], - opts?: { - tickRetryStrategy?: RetryStrategy - updateRetryStrategy?: RetryStrategy - invalidateRetryStrategy?: RetryStrategy - }, + public readonly parents: Indexer[], + options?: IndexerOptions, ) { this.logger = this.logger.for(this) this.state = getInitialState(parents.length) @@ -140,11 +142,11 @@ export abstract class BaseIndexer { }) this.tickRetryStrategy = - opts?.tickRetryStrategy ?? BaseIndexer.GET_DEFAULT_RETRY_STRATEGY() + options?.tickRetryStrategy ?? Indexer.GET_DEFAULT_RETRY_STRATEGY() this.updateRetryStrategy = - opts?.updateRetryStrategy ?? BaseIndexer.GET_DEFAULT_RETRY_STRATEGY() + options?.updateRetryStrategy ?? Indexer.GET_DEFAULT_RETRY_STRATEGY() this.invalidateRetryStrategy = - opts?.invalidateRetryStrategy ?? BaseIndexer.GET_DEFAULT_RETRY_STRATEGY() + options?.invalidateRetryStrategy ?? Indexer.GET_DEFAULT_RETRY_STRATEGY() } get safeHeight(): number | null { @@ -162,20 +164,20 @@ export abstract class BaseIndexer { }) } - subscribe(child: BaseIndexer): void { + subscribe(child: Indexer): void { assert(!this.started, 'Indexer already started') this.logger.debug('Child subscribed', { child: child.constructor.name }) this.children.push(child) } - notifyReady(child: BaseIndexer): void { + notifyReady(child: Indexer): void { this.logger.debug('Someone is ready', { child: child.constructor.name }) const index = this.children.indexOf(child) assert(index !== -1, 'Received ready from unknown child') this.dispatch({ type: 'ChildReady', index }) } - notifyUpdate(parent: BaseIndexer, safeHeight: number | null): void { + notifyUpdate(parent: Indexer, safeHeight: number | null): void { this.logger.debug('Someone has updated', { parent: parent.constructor.name, }) diff --git a/packages/uif/src/index.ts b/packages/uif/src/index.ts index 181838a4..55c38dea 100644 --- a/packages/uif/src/index.ts +++ b/packages/uif/src/index.ts @@ -1,7 +1,7 @@ -export * from './BaseIndexer' export * from './height' +export * from './Indexer' export * from './indexers/ChildIndexer' export * from './indexers/multi/MultiIndexer' -export type { Configuration } from './indexers/multi/types' +export * from './indexers/multi/types' export * from './indexers/RootIndexer' export * from './Retries' diff --git a/packages/uif/src/indexers/ChildIndexer.ts b/packages/uif/src/indexers/ChildIndexer.ts index eedaad97..7d4e2d53 100644 --- a/packages/uif/src/indexers/ChildIndexer.ts +++ b/packages/uif/src/indexers/ChildIndexer.ts @@ -1,6 +1,6 @@ -import { BaseIndexer } from '../BaseIndexer' +import { Indexer } from '../Indexer' -export abstract class ChildIndexer extends BaseIndexer { +export abstract class ChildIndexer extends Indexer { override async tick(): Promise { return Promise.reject(new Error('ChildIndexer cannot tick')) } diff --git a/packages/uif/src/indexers/RootIndexer.ts b/packages/uif/src/indexers/RootIndexer.ts index bae51072..923964a8 100644 --- a/packages/uif/src/indexers/RootIndexer.ts +++ b/packages/uif/src/indexers/RootIndexer.ts @@ -1,10 +1,9 @@ import { Logger } from '@l2beat/backend-tools' -import { BaseIndexer } from '../BaseIndexer' -import { RetryStrategy } from '../Retries' +import { Indexer, IndexerOptions } from '../Indexer' -export abstract class RootIndexer extends BaseIndexer { - constructor(logger: Logger, opts?: { tickRetryStrategy?: RetryStrategy }) { +export abstract class RootIndexer extends Indexer { + constructor(logger: Logger, opts?: IndexerOptions) { super(logger, [], opts) } diff --git a/packages/uif/src/indexers/multi/MultiIndexer.test.ts b/packages/uif/src/indexers/multi/MultiIndexer.test.ts index 743b83d6..8e835dcc 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.test.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.test.ts @@ -300,12 +300,12 @@ describe(MultiIndexer.name, () => { class TestMultiIndexer extends MultiIndexer { constructor( configurations: Configuration[], - private readonly _saved: SavedConfiguration[], + private readonly _saved: SavedConfiguration[], ) { super(Logger.SILENT, [], configurations) } - override multiInitialize(): Promise { + override multiInitialize(): Promise[]> { return Promise.resolve(this._saved) } @@ -324,7 +324,7 @@ function actual(id: string, minHeight: number, maxHeight: number | null) { } function saved(id: string, minHeight: number, currentHeight: number) { - return { id, minHeight, currentHeight } + return { id, properties: null, minHeight, currentHeight } } function update( @@ -341,5 +341,5 @@ function removal( fromHeightInclusive: number, toHeightInclusive: number, ) { - return { id, fromHeightInclusive, toHeightInclusive } + return { id, properties: null, fromHeightInclusive, toHeightInclusive } } diff --git a/packages/uif/src/indexers/multi/MultiIndexer.ts b/packages/uif/src/indexers/multi/MultiIndexer.ts index 2667b709..63084c44 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.ts @@ -1,8 +1,7 @@ import { Logger } from '@l2beat/backend-tools' -import { BaseIndexer } from '../../BaseIndexer' import { Height } from '../../height' -import { RetryStrategy } from '../../Retries' +import { Indexer, IndexerOptions } from '../../Indexer' import { ChildIndexer } from '../ChildIndexer' import { diffConfigurations } from './diffConfigurations' import { toRanges } from './toRanges' @@ -16,18 +15,15 @@ import { export abstract class MultiIndexer extends ChildIndexer { private readonly ranges: ConfigurationRange[] - private saved: SavedConfiguration[] = [] + private saved: SavedConfiguration[] = [] constructor( logger: Logger, - parents: BaseIndexer[], + parents: Indexer[], readonly configurations: Configuration[], - opts?: { - updateRetryStrategy?: RetryStrategy - invalidateRetryStrategy?: RetryStrategy - }, + options?: IndexerOptions, ) { - super(logger, parents, opts) + super(logger, parents, options) this.ranges = toRanges(configurations) } @@ -40,7 +36,7 @@ export abstract class MultiIndexer extends ChildIndexer { * previously with `setStoredConfigurations`. It shouldn't call * `setStoredConfigurations` itself. */ - abstract multiInitialize(): Promise + abstract multiInitialize(): Promise[]> /** * Implements the main data fetching process. It is up to the indexer to @@ -83,7 +79,7 @@ export abstract class MultiIndexer extends ChildIndexer { * This method can only be called during the initialization of the indexer, * after `multiInitialize` returns. */ - abstract removeData(configurations: RemovalConfiguration[]): Promise + abstract removeData(configurations: RemovalConfiguration[]): Promise /** * Saves configurations that the indexer should use to sync data. The @@ -94,7 +90,7 @@ export abstract class MultiIndexer extends ChildIndexer { * configurations are persisted. */ abstract saveConfigurations( - configurations: SavedConfiguration[], + configurations: SavedConfiguration[], ): Promise async initialize(): Promise { @@ -177,7 +173,7 @@ function findRange( function getConfigurationsInRange( range: ConfigurationRange, - savedConfigurations: SavedConfiguration[], + savedConfigurations: SavedConfiguration[], currentHeight: number, ): { configurations: UpdateConfiguration[]; minCurrentHeight: number } { let minCurrentHeight = Infinity @@ -195,9 +191,9 @@ function getConfigurationsInRange( return { configurations, minCurrentHeight } } -function updateSavedConfigurations( - savedConfigurations: SavedConfiguration[], - updatedConfigurations: UpdateConfiguration[], +function updateSavedConfigurations( + savedConfigurations: SavedConfiguration[], + updatedConfigurations: UpdateConfiguration[], newHeight: number, ): void { for (const updated of updatedConfigurations) { @@ -207,6 +203,7 @@ function updateSavedConfigurations( } else { savedConfigurations.push({ id: updated.id, + properties: updated.properties, minHeight: updated.minHeight, currentHeight: newHeight, }) diff --git a/packages/uif/src/indexers/multi/diffConfigurations.test.ts b/packages/uif/src/indexers/multi/diffConfigurations.test.ts index b9c9694c..eebfddf0 100644 --- a/packages/uif/src/indexers/multi/diffConfigurations.test.ts +++ b/packages/uif/src/indexers/multi/diffConfigurations.test.ts @@ -184,7 +184,7 @@ function actual(id: string, minHeight: number, maxHeight: number | null) { } function saved(id: string, minHeight: number, currentHeight: number) { - return { id, minHeight, currentHeight } + return { id, properties: null, minHeight, currentHeight } } function removal( @@ -192,5 +192,5 @@ function removal( fromHeightInclusive: number, toHeightInclusive: number, ) { - return { id, fromHeightInclusive, toHeightInclusive } + return { id, properties: null, fromHeightInclusive, toHeightInclusive } } diff --git a/packages/uif/src/indexers/multi/diffConfigurations.ts b/packages/uif/src/indexers/multi/diffConfigurations.ts index 7c71e5fb..c4f47ae5 100644 --- a/packages/uif/src/indexers/multi/diffConfigurations.ts +++ b/packages/uif/src/indexers/multi/diffConfigurations.ts @@ -5,12 +5,12 @@ import { SavedConfiguration, } from './types' -export function diffConfigurations( - actual: Configuration[], - saved: SavedConfiguration[], +export function diffConfigurations( + actual: Configuration[], + saved: SavedConfiguration[], ): { - toRemove: RemovalConfiguration[] - toSave: SavedConfiguration[] + toRemove: RemovalConfiguration[] + toSave: SavedConfiguration[] safeHeight: number | null } { let safeHeight: number | null = Infinity @@ -19,10 +19,11 @@ export function diffConfigurations( const actualMap = new Map(actual.map((c) => [c.id, c])) const savedMap = new Map(saved.map((c) => [c.id, c])) - const toRemove: RemovalConfiguration[] = saved + const toRemove: RemovalConfiguration[] = saved .filter((c) => !actualMap.has(c.id)) .map((c) => ({ id: c.id, + properties: c.properties, fromHeightInclusive: c.minHeight, toHeightInclusive: c.currentHeight, })) @@ -51,12 +52,14 @@ export function diffConfigurations( // We will re-download everything from the beginning toRemove.push({ id: stored.id, + properties: stored.properties, fromHeightInclusive: stored.minHeight, toHeightInclusive: stored.currentHeight, }) } else if (stored.minHeight < c.minHeight) { toRemove.push({ id: stored.id, + properties: stored.properties, fromHeightInclusive: stored.minHeight, toHeightInclusive: c.minHeight - 1, }) @@ -65,6 +68,7 @@ export function diffConfigurations( if (c.maxHeight !== null && stored.currentHeight > c.maxHeight) { toRemove.push({ id: stored.id, + properties: stored.properties, fromHeightInclusive: c.maxHeight + 1, toHeightInclusive: stored.currentHeight, }) @@ -77,13 +81,14 @@ export function diffConfigurations( } const toSave = saved - .map((c): SavedConfiguration | undefined => { + .map((c): SavedConfiguration | undefined => { const actual = actualMap.get(c.id) if (!actual || actual.minHeight < c.minHeight) { return undefined } return { id: c.id, + properties: c.properties, minHeight: actual.minHeight, currentHeight: actual.maxHeight === null @@ -91,7 +96,7 @@ export function diffConfigurations( : Math.min(c.currentHeight, actual.maxHeight), } }) - .filter((c): c is SavedConfiguration => c !== undefined) + .filter((c): c is SavedConfiguration => c !== undefined) return { toRemove, toSave, safeHeight } } diff --git a/packages/uif/src/indexers/multi/types.ts b/packages/uif/src/indexers/multi/types.ts index a372ad78..0d7e5834 100644 --- a/packages/uif/src/indexers/multi/types.ts +++ b/packages/uif/src/indexers/multi/types.ts @@ -9,14 +9,16 @@ export interface UpdateConfiguration extends Configuration { hasData: boolean } -export interface SavedConfiguration { +export interface SavedConfiguration { id: string + properties: T minHeight: number currentHeight: number } -export interface RemovalConfiguration { +export interface RemovalConfiguration { id: string + properties: T fromHeightInclusive: number toHeightInclusive: number } From e189957059efbb542b113f859966b860d4eb9fb7 Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Thu, 21 Mar 2024 23:57:22 +0100 Subject: [PATCH 17/31] Simplify the code by only counting full hours --- packages/uif-example/src/Application.ts | 7 ++++--- packages/uif-example/src/HourlyIndexer.ts | 8 ++++--- .../uif-example/src/prices/PriceIndexer.ts | 21 ++++++------------- .../uif-example/src/prices/PriceService.ts | 4 ++-- packages/uif-example/src/utils.ts | 9 ++++++++ 5 files changed, 26 insertions(+), 23 deletions(-) create mode 100644 packages/uif-example/src/utils.ts diff --git a/packages/uif-example/src/Application.ts b/packages/uif-example/src/Application.ts index cb7aa8e7..7a196e70 100644 --- a/packages/uif-example/src/Application.ts +++ b/packages/uif-example/src/Application.ts @@ -5,6 +5,7 @@ import { PriceIndexer } from './prices/PriceIndexer' import { PriceIndexerRepository } from './prices/PriceIndexerRepository' import { PriceRepository } from './prices/PriceRepository' import { PriceService } from './prices/PriceService' +import { msToHours } from './utils' export class Application { start: () => Promise @@ -30,13 +31,13 @@ export class Application { // could be a hash of properties & minHeight instead id: 'eth-ethereum', properties: { tokenSymbol: 'ETH', apiId: 'ethereum' }, - minHeight: new Date('2021-01-01T00:00:00Z').getTime(), + minHeight: msToHours(new Date('2021-01-01T00:00:00Z').getTime()), maxHeight: null, }, { id: 'weth-ethereum', properties: { tokenSymbol: 'WETH', apiId: 'ethereum' }, - minHeight: new Date('2022-01-01T00:00:00Z').getTime(), + minHeight: msToHours(new Date('2022-01-01T00:00:00Z').getTime()), maxHeight: null, }, ], @@ -52,7 +53,7 @@ export class Application { { id: 'btc-bitcoin', properties: { tokenSymbol: 'BTC', apiId: 'bitcoin' }, - minHeight: new Date('2022-01-01T00:00:00Z').getTime(), + minHeight: msToHours(new Date('2022-01-01T00:00:00Z').getTime()), maxHeight: null, }, ], diff --git a/packages/uif-example/src/HourlyIndexer.ts b/packages/uif-example/src/HourlyIndexer.ts index 34eb6436..46a7245e 100644 --- a/packages/uif-example/src/HourlyIndexer.ts +++ b/packages/uif-example/src/HourlyIndexer.ts @@ -1,5 +1,7 @@ import { RootIndexer } from '@l2beat/uif' +import { ONE_HOUR_MS } from './utils' + export class HourlyIndexer extends RootIndexer { async initialize(): Promise { setInterval(() => this.requestTick(), 60 * 1000) @@ -7,8 +9,8 @@ export class HourlyIndexer extends RootIndexer { } async tick(): Promise { - const hourInMs = 60 * 60 * 1000 - const time = (new Date().getTime() % hourInMs) * hourInMs - return Promise.resolve(time) + const now = new Date().getTime() + const hours = Math.floor(now / ONE_HOUR_MS) + return Promise.resolve(hours) } } diff --git a/packages/uif-example/src/prices/PriceIndexer.ts b/packages/uif-example/src/prices/PriceIndexer.ts index 3e9c42b0..f9fddcf8 100644 --- a/packages/uif-example/src/prices/PriceIndexer.ts +++ b/packages/uif-example/src/prices/PriceIndexer.ts @@ -9,13 +9,12 @@ import { UpdateConfiguration, } from '@l2beat/uif' +import { ONE_HOUR_MS } from '../utils' import { PriceConfig } from './PriceConfig' import { PriceIndexerRepository } from './PriceIndexerRepository' import { PriceRepository } from './PriceRepository' import { PriceService } from './PriceService' -const ONE_HOUR = 60 * 60 * 1000 - export class PriceIndexer extends MultiIndexer { private readonly apiId: string @@ -49,21 +48,14 @@ export class PriceIndexer extends MultiIndexer { targetHeight: number, configurations: UpdateConfiguration[], ): Promise { - const startHour = currentHeight - (currentHeight % ONE_HOUR) + ONE_HOUR - const endHour = Math.min( - targetHeight - (targetHeight % ONE_HOUR), - // for example the api costs us more money for larger ranges - startHour + 23 * ONE_HOUR, - ) - - if (startHour >= endHour) { - return startHour - } + const startHour = currentHeight + 1 + // we only query 24 hours at a time + const endHour = Math.min(targetHeight, startHour + 23) const prices = await this.priceService.getHourlyPrices( this.apiId, - startHour, - endHour, + startHour * ONE_HOUR_MS, + endHour * ONE_HOUR_MS, ) const dataToSave = configurations.flatMap((configuration) => { @@ -75,7 +67,6 @@ export class PriceIndexer extends MultiIndexer { }) await this.priceRepository.save(dataToSave) - // TODO: Maybe if targetHeight is not exactly an hour we can return it? return endHour } diff --git a/packages/uif-example/src/prices/PriceService.ts b/packages/uif-example/src/prices/PriceService.ts index 759ec11f..426bbc96 100644 --- a/packages/uif-example/src/prices/PriceService.ts +++ b/packages/uif-example/src/prices/PriceService.ts @@ -1,4 +1,4 @@ -const ONE_HOUR = 60 * 60 * 1000 +import { ONE_HOUR_MS } from '../utils' export class PriceService { async getHourlyPrices( @@ -9,7 +9,7 @@ export class PriceService { apiId // use it so that eslint doesn't complain const prices: { timestamp: number; price: number }[] = [] - for (let t = startHourInclusive; t <= endHourInclusive; t += ONE_HOUR) { + for (let t = startHourInclusive; t <= endHourInclusive; t += ONE_HOUR_MS) { prices.push({ timestamp: t, price: Math.random() * 1000 }) } return Promise.resolve(prices) diff --git a/packages/uif-example/src/utils.ts b/packages/uif-example/src/utils.ts new file mode 100644 index 00000000..55eab79d --- /dev/null +++ b/packages/uif-example/src/utils.ts @@ -0,0 +1,9 @@ +export const ONE_HOUR_MS = 60 * 60 * 1000 + +export function msToHours(timestampOrDate: number | Date): number { + const timestamp = + typeof timestampOrDate === 'number' + ? timestampOrDate + : timestampOrDate.getTime() + return Math.floor(timestamp / ONE_HOUR_MS) +} From b68a59bcc0dd19d9d32c3602964121f372f9297d Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Fri, 22 Mar 2024 00:21:28 +0100 Subject: [PATCH 18/31] Do not propagate repeated ticks --- packages/uif-example/src/Application.ts | 10 +++---- packages/uif-example/src/HourlyIndexer.ts | 2 +- .../uif-example/src/prices/PriceIndexer.ts | 20 ++++++++------ .../uif-example/src/prices/PriceRepository.ts | 27 ++++++++++++++++--- .../uif-example/src/prices/PriceService.ts | 19 ++++++++++--- .../reducer/handlers/handleTickSucceeded.ts | 7 ++--- .../uif/src/reducer/indexerReducer.test.ts | 22 +++++++++++++++ 7 files changed, 83 insertions(+), 24 deletions(-) diff --git a/packages/uif-example/src/Application.ts b/packages/uif-example/src/Application.ts index 7a196e70..aa076601 100644 --- a/packages/uif-example/src/Application.ts +++ b/packages/uif-example/src/Application.ts @@ -5,7 +5,7 @@ import { PriceIndexer } from './prices/PriceIndexer' import { PriceIndexerRepository } from './prices/PriceIndexerRepository' import { PriceRepository } from './prices/PriceRepository' import { PriceService } from './prices/PriceService' -import { msToHours } from './utils' +import { msToHours, ONE_HOUR_MS } from './utils' export class Application { start: () => Promise @@ -15,7 +15,7 @@ export class Application { const hourlyIndexer = new HourlyIndexer(logger) - const priceService = new PriceService() + const priceService = new PriceService(logger) const priceRepository = new PriceRepository() const priceIndexerRepository = new PriceIndexerRepository() @@ -31,13 +31,13 @@ export class Application { // could be a hash of properties & minHeight instead id: 'eth-ethereum', properties: { tokenSymbol: 'ETH', apiId: 'ethereum' }, - minHeight: msToHours(new Date('2021-01-01T00:00:00Z').getTime()), + minHeight: msToHours(Date.now() - 48 * ONE_HOUR_MS), maxHeight: null, }, { id: 'weth-ethereum', properties: { tokenSymbol: 'WETH', apiId: 'ethereum' }, - minHeight: msToHours(new Date('2022-01-01T00:00:00Z').getTime()), + minHeight: msToHours(Date.now() - 32 * ONE_HOUR_MS), maxHeight: null, }, ], @@ -53,7 +53,7 @@ export class Application { { id: 'btc-bitcoin', properties: { tokenSymbol: 'BTC', apiId: 'bitcoin' }, - minHeight: msToHours(new Date('2022-01-01T00:00:00Z').getTime()), + minHeight: msToHours(Date.now() - 72 * ONE_HOUR_MS), maxHeight: null, }, ], diff --git a/packages/uif-example/src/HourlyIndexer.ts b/packages/uif-example/src/HourlyIndexer.ts index 46a7245e..6093b8cf 100644 --- a/packages/uif-example/src/HourlyIndexer.ts +++ b/packages/uif-example/src/HourlyIndexer.ts @@ -4,7 +4,7 @@ import { ONE_HOUR_MS } from './utils' export class HourlyIndexer extends RootIndexer { async initialize(): Promise { - setInterval(() => this.requestTick(), 60 * 1000) + setInterval(() => this.requestTick(), 10 * 1000) return this.tick() } diff --git a/packages/uif-example/src/prices/PriceIndexer.ts b/packages/uif-example/src/prices/PriceIndexer.ts index f9fddcf8..9f923562 100644 --- a/packages/uif-example/src/prices/PriceIndexer.ts +++ b/packages/uif-example/src/prices/PriceIndexer.ts @@ -29,14 +29,7 @@ export class PriceIndexer extends MultiIndexer { options?: IndexerOptions, ) { super(logger, parents, configurations, options) - const apiId = configurations[0]?.properties.apiId - if (!apiId) { - throw new Error('At least one configuration is required') - } - if (configurations.some((c) => c.properties.apiId !== apiId)) { - throw new Error('All configurations must have the same apiId') - } - this.apiId = apiId + this.apiId = getCommonApiId(configurations) } override async multiInitialize(): Promise[]> { @@ -88,3 +81,14 @@ export class PriceIndexer extends MultiIndexer { return this.priceIndexerRepository.save(this.indexerId, configurations) } } + +function getCommonApiId(configurations: Configuration[]): string { + const apiId = configurations[0]?.properties.apiId + if (!apiId) { + throw new Error('At least one configuration is required') + } + if (configurations.some((c) => c.properties.apiId !== apiId)) { + throw new Error('All configurations must have the same apiId') + } + return apiId +} diff --git a/packages/uif-example/src/prices/PriceRepository.ts b/packages/uif-example/src/prices/PriceRepository.ts index aafa9d71..2690a32c 100644 --- a/packages/uif-example/src/prices/PriceRepository.ts +++ b/packages/uif-example/src/prices/PriceRepository.ts @@ -1,8 +1,16 @@ export class PriceRepository { + data = new Map>() + async save( prices: { tokenSymbol: string; timestamp: number; price: number }[], ): Promise { - prices // use it so that eslint doesn't complain + for (const { tokenSymbol, timestamp, price } of prices) { + const tokenPrices = + this.data.get(tokenSymbol) ?? new Map() + this.data.set(tokenSymbol, tokenPrices) + + tokenPrices.set(timestamp, price) + } return Promise.resolve() } @@ -11,9 +19,20 @@ export class PriceRepository { fromTimestampInclusive: number, toTimestampInclusive: number, ): Promise { - tokenSymbol // use it so that eslint doesn't complain - fromTimestampInclusive // use it so that eslint doesn't complain - toTimestampInclusive // use it so that eslint doesn't complain + const tokenPrices = this.data.get(tokenSymbol) + if (!tokenPrices) { + return + } + + for (const [timestamp] of tokenPrices) { + if ( + timestamp >= fromTimestampInclusive && + timestamp <= toTimestampInclusive + ) { + tokenPrices.delete(timestamp) + } + } + return Promise.resolve() } } diff --git a/packages/uif-example/src/prices/PriceService.ts b/packages/uif-example/src/prices/PriceService.ts index 426bbc96..0026a094 100644 --- a/packages/uif-example/src/prices/PriceService.ts +++ b/packages/uif-example/src/prices/PriceService.ts @@ -1,17 +1,30 @@ +import { Logger } from '@l2beat/backend-tools' +import { setTimeout } from 'timers/promises' + import { ONE_HOUR_MS } from '../utils' export class PriceService { + constructor(private readonly logger: Logger) { + this.logger = logger.for(this) + } + async getHourlyPrices( apiId: string, startHourInclusive: number, endHourInclusive: number, ): Promise<{ timestamp: number; price: number }[]> { - apiId // use it so that eslint doesn't complain - const prices: { timestamp: number; price: number }[] = [] for (let t = startHourInclusive; t <= endHourInclusive; t += ONE_HOUR_MS) { prices.push({ timestamp: t, price: Math.random() * 1000 }) } - return Promise.resolve(prices) + + await setTimeout(1000) + + this.logger.info('Fetched prices', { + apiId, + since: startHourInclusive, + count: prices.length, + }) + return prices } } diff --git a/packages/uif/src/reducer/handlers/handleTickSucceeded.ts b/packages/uif/src/reducer/handlers/handleTickSucceeded.ts index c3b41e91..c1fc15d0 100644 --- a/packages/uif/src/reducer/handlers/handleTickSucceeded.ts +++ b/packages/uif/src/reducer/handlers/handleTickSucceeded.ts @@ -11,9 +11,10 @@ export function handleTickSucceeded( ): IndexerReducerResult { assertRoot(state) assertStatus(state.status, 'ticking') - const effects: IndexerEffect[] = [ - { type: 'SetSafeHeight', safeHeight: action.safeHeight }, - ] + const effects: IndexerEffect[] = + action.safeHeight !== state.safeHeight + ? [{ type: 'SetSafeHeight', safeHeight: action.safeHeight }] + : [] if (state.tickScheduled) { effects.push({ type: 'Tick' }) } diff --git a/packages/uif/src/reducer/indexerReducer.test.ts b/packages/uif/src/reducer/indexerReducer.test.ts index e5506b63..48f7812b 100644 --- a/packages/uif/src/reducer/indexerReducer.test.ts +++ b/packages/uif/src/reducer/indexerReducer.test.ts @@ -493,6 +493,7 @@ describe(indexerReducer.name, () => { { type: 'Invalidate', targetHeight: 140 }, ]) }) + it('if partially invalidating, does not update until done invalidating fully', () => { const initState = getAfterInit({ safeHeight: 100, @@ -625,6 +626,27 @@ describe(indexerReducer.name, () => { { type: 'Tick' }, ]) }) + + it('does not save the same safe height', () => { + const initState = getInitialState(0) + const [state, effects] = reduceWithIndexerReducer(initState, [ + { type: 'Initialized', safeHeight: 100, childCount: 0 }, + { type: 'RequestTick' }, + { type: 'TickSucceeded', safeHeight: 100 }, + { type: 'RequestTick' }, + { type: 'TickSucceeded', safeHeight: 100 }, + ]) + + expect(state).toEqual({ + ...initState, + status: 'idle', + height: 100, + safeHeight: 100, + invalidateToHeight: 100, + initializedSelf: true, + }) + expect(effects).toEqual([]) + }) }) describe('fatal errors', () => { From 9835b36d2b3eba0ffbaa5aae386e3e5412323f01 Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Fri, 22 Mar 2024 00:43:41 +0100 Subject: [PATCH 19/31] Add BlockIndexer example --- packages/uif-example/src/Application.ts | 21 +++++++- .../uif-example/src/blocks/BlockIndexer.ts | 52 +++++++++++++++++++ .../src/blocks/BlockIndexerRepository.ts | 12 +++++ .../uif-example/src/blocks/BlockRepository.ts | 8 +++ .../uif-example/src/blocks/BlockService.ts | 8 +++ .../uif-example/src/prices/PriceIndexer.ts | 6 +-- 6 files changed, 102 insertions(+), 5 deletions(-) create mode 100644 packages/uif-example/src/blocks/BlockIndexer.ts create mode 100644 packages/uif-example/src/blocks/BlockIndexerRepository.ts create mode 100644 packages/uif-example/src/blocks/BlockRepository.ts create mode 100644 packages/uif-example/src/blocks/BlockService.ts diff --git a/packages/uif-example/src/Application.ts b/packages/uif-example/src/Application.ts index aa076601..3088053d 100644 --- a/packages/uif-example/src/Application.ts +++ b/packages/uif-example/src/Application.ts @@ -1,5 +1,9 @@ import { Logger } from '@l2beat/backend-tools' +import { BlockIndexer } from './blocks/BlockIndexer' +import { BlockIndexerRepository } from './blocks/BlockIndexerRepository' +import { BlockRepository } from './blocks/BlockRepository' +import { BlockService } from './blocks/BlockService' import { HourlyIndexer } from './HourlyIndexer' import { PriceIndexer } from './prices/PriceIndexer' import { PriceIndexerRepository } from './prices/PriceIndexerRepository' @@ -24,8 +28,8 @@ export class Application { priceService, priceRepository, priceIndexerRepository, + hourlyIndexer, logger, - [hourlyIndexer], [ { // could be a hash of properties & minHeight instead @@ -47,8 +51,8 @@ export class Application { priceService, priceRepository, priceIndexerRepository, + hourlyIndexer, logger, - [hourlyIndexer], [ { id: 'btc-bitcoin', @@ -59,12 +63,25 @@ export class Application { ], ) + const blockService = new BlockService() + const blockRepository = new BlockRepository() + const blockIndexerRepository = new BlockIndexerRepository() + const blockIndexer = new BlockIndexer( + blockService, + blockRepository, + blockIndexerRepository, + msToHours(Date.now() - 72 * ONE_HOUR_MS), + hourlyIndexer, + logger, + ) + this.start = async (): Promise => { logger.for('Application').info('Starting') await hourlyIndexer.start() await ethereumPriceIndexer.start() await bitcoinPriceIndexer.start() + await blockIndexer.start() logger.for('Application').info('Started') } diff --git a/packages/uif-example/src/blocks/BlockIndexer.ts b/packages/uif-example/src/blocks/BlockIndexer.ts new file mode 100644 index 00000000..6a8e532f --- /dev/null +++ b/packages/uif-example/src/blocks/BlockIndexer.ts @@ -0,0 +1,52 @@ +import { Logger } from '@l2beat/backend-tools' +import { ChildIndexer, IndexerOptions } from '@l2beat/uif' + +import { HourlyIndexer } from '../HourlyIndexer' +import { ONE_HOUR_MS } from '../utils' +import { BlockIndexerRepository } from './BlockIndexerRepository' +import { BlockRepository } from './BlockRepository' +import { BlockService } from './BlockService' + +export class BlockIndexer extends ChildIndexer { + constructor( + private readonly blockService: BlockService, + private readonly blockRepository: BlockRepository, + private readonly blockIndexerRepository: BlockIndexerRepository, + private readonly minHeight: number, + hourlyIndexer: HourlyIndexer, + logger: Logger, + options?: IndexerOptions, + ) { + super(logger, [hourlyIndexer], options) + } + + override async initialize(): Promise { + const height = await this.blockIndexerRepository.loadHeight() + return height ?? null + } + + override async setSafeHeight(height: number | null): Promise { + await this.blockIndexerRepository.saveHeight(height ?? undefined) + } + + override async update( + currentHeight: number | null, + _targetHeight: number, + ): Promise { + const nextHeight = + currentHeight === null ? this.minHeight : currentHeight + 1 + const timestamp = nextHeight * ONE_HOUR_MS + + const block = await this.blockService.getBlockNumberBefore(timestamp) + await this.blockRepository.save({ number: block, timestamp }) + + return nextHeight + } + + override async invalidate( + targetHeight: number | null, + ): Promise { + // We don't need to delete any data + return Promise.resolve(targetHeight) + } +} diff --git a/packages/uif-example/src/blocks/BlockIndexerRepository.ts b/packages/uif-example/src/blocks/BlockIndexerRepository.ts new file mode 100644 index 00000000..2f18979d --- /dev/null +++ b/packages/uif-example/src/blocks/BlockIndexerRepository.ts @@ -0,0 +1,12 @@ +export class BlockIndexerRepository { + private height: number | undefined + + async loadHeight(): Promise { + return Promise.resolve(this.height) + } + + async saveHeight(height: number | undefined): Promise { + this.height = height + return Promise.resolve() + } +} diff --git a/packages/uif-example/src/blocks/BlockRepository.ts b/packages/uif-example/src/blocks/BlockRepository.ts new file mode 100644 index 00000000..f046278c --- /dev/null +++ b/packages/uif-example/src/blocks/BlockRepository.ts @@ -0,0 +1,8 @@ +export class BlockRepository { + private readonly blocks = new Map() + + async save(block: { number: number; timestamp: number }): Promise { + this.blocks.set(block.timestamp, block.number) + return Promise.resolve() + } +} diff --git a/packages/uif-example/src/blocks/BlockService.ts b/packages/uif-example/src/blocks/BlockService.ts new file mode 100644 index 00000000..d9825c72 --- /dev/null +++ b/packages/uif-example/src/blocks/BlockService.ts @@ -0,0 +1,8 @@ +import { setTimeout } from 'timers/promises' + +export class BlockService { + async getBlockNumberBefore(timestamp: number): Promise { + await setTimeout(200) + return Math.floor(timestamp / 123456) + } +} diff --git a/packages/uif-example/src/prices/PriceIndexer.ts b/packages/uif-example/src/prices/PriceIndexer.ts index 9f923562..4e6f9dbf 100644 --- a/packages/uif-example/src/prices/PriceIndexer.ts +++ b/packages/uif-example/src/prices/PriceIndexer.ts @@ -1,7 +1,6 @@ import { Logger } from '@l2beat/backend-tools' import { Configuration, - Indexer, IndexerOptions, MultiIndexer, RemovalConfiguration, @@ -9,6 +8,7 @@ import { UpdateConfiguration, } from '@l2beat/uif' +import { HourlyIndexer } from '../HourlyIndexer' import { ONE_HOUR_MS } from '../utils' import { PriceConfig } from './PriceConfig' import { PriceIndexerRepository } from './PriceIndexerRepository' @@ -23,12 +23,12 @@ export class PriceIndexer extends MultiIndexer { private readonly priceService: PriceService, private readonly priceRepository: PriceRepository, private readonly priceIndexerRepository: PriceIndexerRepository, + hourlyIndexer: HourlyIndexer, logger: Logger, - parents: Indexer[], configurations: Configuration[], options?: IndexerOptions, ) { - super(logger, parents, configurations, options) + super(logger, [hourlyIndexer], configurations, options) this.apiId = getCommonApiId(configurations) } From a7d255a76ede2182886042e97a29e7c21c19a9f5 Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Fri, 22 Mar 2024 12:13:12 +0100 Subject: [PATCH 20/31] Add some todos --- .../uif-example/src/prices/PriceIndexer.ts | 18 +++++++++++------- .../src/indexers/multi/MultiIndexer.test.ts | 1 + .../uif/src/indexers/multi/MultiIndexer.ts | 7 ++++--- packages/uif/src/indexers/multi/types.ts | 8 +++++++- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/packages/uif-example/src/prices/PriceIndexer.ts b/packages/uif-example/src/prices/PriceIndexer.ts index 4e6f9dbf..467d3026 100644 --- a/packages/uif-example/src/prices/PriceIndexer.ts +++ b/packages/uif-example/src/prices/PriceIndexer.ts @@ -51,13 +51,17 @@ export class PriceIndexer extends MultiIndexer { endHour * ONE_HOUR_MS, ) - const dataToSave = configurations.flatMap((configuration) => { - return prices.map(({ timestamp, price }) => ({ - tokenSymbol: configuration.properties.tokenSymbol, - timestamp, - price, - })) - }) + const dataToSave = configurations + // TODO: don't update currentHeight for configs that have data + // TODO: test data downloaded to middle of the range + .filter((c) => !c.hasData) + .flatMap((configuration) => { + return prices.map(({ timestamp, price }) => ({ + tokenSymbol: configuration.properties.tokenSymbol, + timestamp, + price, + })) + }) await this.priceRepository.save(dataToSave) return endHour diff --git a/packages/uif/src/indexers/multi/MultiIndexer.test.ts b/packages/uif/src/indexers/multi/MultiIndexer.test.ts index 8e835dcc..39468314 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.test.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.test.ts @@ -196,6 +196,7 @@ describe(MultiIndexer.name, () => { ]) // The same range. In real life might be a result of a parent reorg + // Invalidate is a no-op so we don't need to call it expect(await testIndexer.update(100, 500)).toEqual(200) expect(testIndexer.multiUpdate).toHaveBeenNthCalledWith(2, 100, 200, [ update('a', 100, 200, true), diff --git a/packages/uif/src/indexers/multi/MultiIndexer.ts b/packages/uif/src/indexers/multi/MultiIndexer.ts index 63084c44..33d0caf0 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.ts @@ -198,15 +198,16 @@ function updateSavedConfigurations( ): void { for (const updated of updatedConfigurations) { const saved = savedConfigurations.find((c) => c.id === updated.id) - if (saved) { - saved.currentHeight = newHeight - } else { + if (!saved) { savedConfigurations.push({ id: updated.id, properties: updated.properties, minHeight: updated.minHeight, currentHeight: newHeight, }) + } else { + // TODO: test this + saved.currentHeight = Math.max(saved.currentHeight, newHeight) } } } diff --git a/packages/uif/src/indexers/multi/types.ts b/packages/uif/src/indexers/multi/types.ts index 0d7e5834..edd807d3 100644 --- a/packages/uif/src/indexers/multi/types.ts +++ b/packages/uif/src/indexers/multi/types.ts @@ -5,7 +5,11 @@ export interface Configuration { maxHeight: number | null } -export interface UpdateConfiguration extends Configuration { +export interface UpdateConfiguration { + id: string + properties: T + minHeight: number + maxHeight: number | null hasData: boolean } @@ -13,6 +17,8 @@ export interface SavedConfiguration { id: string properties: T minHeight: number + // TODO: add maxHeight + // TODO: add null, save configurations without syncing currentHeight: number } From 54241456eb6211461e1f12060edd891ea72bde87 Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Fri, 22 Mar 2024 12:26:54 +0100 Subject: [PATCH 21/31] Get rid of null heights --- .../uif-example/src/blocks/BlockIndexer.ts | 20 +++------ .../src/blocks/BlockIndexerRepository.ts | 2 +- packages/uif/src/Indexer.ts | 38 +++++++--------- packages/uif/src/height.ts | 45 ------------------- packages/uif/src/index.ts | 1 - .../src/indexers/multi/MultiIndexer.test.ts | 2 +- .../uif/src/indexers/multi/MultiIndexer.ts | 25 +++-------- .../src/indexers/multi/diffConfigurations.ts | 16 +++---- .../reducer/handlers/handleParentUpdated.ts | 4 +- .../reducer/handlers/handleUpdateSucceeded.ts | 8 +--- .../src/reducer/helpers/continueOperations.ts | 21 +++------ .../reducer/helpers/finishInitialization.ts | 5 +-- .../uif/src/reducer/types/IndexerAction.ts | 10 ++--- .../uif/src/reducer/types/IndexerEffect.ts | 4 +- .../uif/src/reducer/types/IndexerState.ts | 8 ++-- 15 files changed, 60 insertions(+), 149 deletions(-) delete mode 100644 packages/uif/src/height.ts diff --git a/packages/uif-example/src/blocks/BlockIndexer.ts b/packages/uif-example/src/blocks/BlockIndexer.ts index 6a8e532f..80ac9410 100644 --- a/packages/uif-example/src/blocks/BlockIndexer.ts +++ b/packages/uif-example/src/blocks/BlockIndexer.ts @@ -20,21 +20,17 @@ export class BlockIndexer extends ChildIndexer { super(logger, [hourlyIndexer], options) } - override async initialize(): Promise { + override async initialize(): Promise { const height = await this.blockIndexerRepository.loadHeight() - return height ?? null + return height ?? this.minHeight - 1 } - override async setSafeHeight(height: number | null): Promise { - await this.blockIndexerRepository.saveHeight(height ?? undefined) + override async setSafeHeight(height: number): Promise { + await this.blockIndexerRepository.saveHeight(height) } - override async update( - currentHeight: number | null, - _targetHeight: number, - ): Promise { - const nextHeight = - currentHeight === null ? this.minHeight : currentHeight + 1 + override async update(currentHeight: number): Promise { + const nextHeight = currentHeight + 1 const timestamp = nextHeight * ONE_HOUR_MS const block = await this.blockService.getBlockNumberBefore(timestamp) @@ -43,9 +39,7 @@ export class BlockIndexer extends ChildIndexer { return nextHeight } - override async invalidate( - targetHeight: number | null, - ): Promise { + override async invalidate(targetHeight: number): Promise { // We don't need to delete any data return Promise.resolve(targetHeight) } diff --git a/packages/uif-example/src/blocks/BlockIndexerRepository.ts b/packages/uif-example/src/blocks/BlockIndexerRepository.ts index 2f18979d..de63934f 100644 --- a/packages/uif-example/src/blocks/BlockIndexerRepository.ts +++ b/packages/uif-example/src/blocks/BlockIndexerRepository.ts @@ -5,7 +5,7 @@ export class BlockIndexerRepository { return Promise.resolve(this.height) } - async saveHeight(height: number | undefined): Promise { + async saveHeight(height: number): Promise { this.height = height return Promise.resolve() } diff --git a/packages/uif/src/Indexer.ts b/packages/uif/src/Indexer.ts index 0a7a8e70..9ef664df 100644 --- a/packages/uif/src/Indexer.ts +++ b/packages/uif/src/Indexer.ts @@ -3,7 +3,6 @@ import { assert } from 'node:console' import { Logger } from '@l2beat/backend-tools' import { assertUnreachable } from './assertUnreachable' -import { Height } from './height' import { getInitialState } from './reducer/getInitialState' import { indexerReducer } from './reducer/indexerReducer' import { IndexerAction } from './reducer/types/IndexerAction' @@ -40,8 +39,8 @@ export abstract class Indexer { /** * Initializes the indexer. It should return a height that the indexer has * synced up to. If the indexer has not synced any data, it should return - * `null`. For root indexers it should return the initial target height for - * the entire system. + * `minHeight - 1`. For root indexers it should return the initial target + * height for the entire system. * * This method is expected to read the height that was saved previously with * `setSafeHeight`. It shouldn't call `setSafeHeight` itself. @@ -51,19 +50,19 @@ export abstract class Indexer { * This method should also schedule a process to request ticks. For example * with `setInterval(() => this.requestTick(), 1000)`. */ - abstract initialize(): Promise + abstract initialize(): Promise /** * Saves the height (most likely to a database). The height given is the * smallest height from all parents and what the indexer itself synced to - * previously. It can be `null`. + * previously. * * When `initialize` is called it is expected that it will read the same * height that was saved here. * * Optional in root indexers. */ - abstract setSafeHeight(height: number | null): Promise + abstract setSafeHeight(height: number): Promise /** * Implements the main data fetching process. It is up to the indexer to @@ -71,9 +70,9 @@ export abstract class Indexer { * indexer can only fetch data up to 110 and return 110. The next time this * method will be called with `.update(110, 200)`. * - * @param currentHeight The height that the indexer has synced up to previously. Can - * be `null` if no data was synced. This value is exclusive so the indexer - * should not fetch data for this height. + * @param currentHeight The height that the indexer has synced up to + * previously. This value is exclusive so the indexer should not fetch data + * for this height. * * @param targetHeight The height that the indexer should sync up to. This value is * inclusive so the indexer should eventually fetch data for this height. @@ -82,13 +81,10 @@ export abstract class Indexer { * `currentHeight` means that the indexer has not synced any data. Returning * a value greater than `currentHeight` means that the indexer has synced up * to that height. Returning a value less than `currentHeight` will trigger - * invalidation down to the returned value. Returning `null` will invalidate - * all data. Returning a value greater than `targetHeight` is not permitted. + * invalidation down to the returned value. Returning a value greater than + * `targetHeight` is not permitted. */ - abstract update( - currentHeight: number | null, - targetHeight: number, - ): Promise + abstract update(currentHeight: number, targetHeight: number): Promise /** * Responsible for invalidating data that was synced previously. It is @@ -102,22 +98,18 @@ export abstract class Indexer { * steps, you can return a height that is larger than the target height. * * @param targetHeight The height that the indexer should invalidate down to. - * Can be `null`. If it is `null`, the indexer should invalidate all - * data. * * @returns The height that the indexer has invalidated down to. Returning * `targetHeight` means that the indexer has invalidated all the required * data. Returning a value greater than `targetHeight` means that the indexer * has invalidated down to that height. */ - abstract invalidate(targetHeight: number | null): Promise + abstract invalidate(targetHeight: number): Promise /** * This method is responsible for providing the target height for the entire * system. Some candidates for this are: the current time or the latest block * number. - * - * This method cannot return `null`. */ abstract tick(): Promise @@ -149,7 +141,7 @@ export abstract class Indexer { options?.invalidateRetryStrategy ?? Indexer.GET_DEFAULT_RETRY_STRATEGY() } - get safeHeight(): number | null { + get safeHeight(): number { return this.state.safeHeight } @@ -177,7 +169,7 @@ export abstract class Indexer { this.dispatch({ type: 'ChildReady', index }) } - notifyUpdate(parent: Indexer, safeHeight: number | null): void { + notifyUpdate(parent: Indexer, safeHeight: number): void { this.logger.debug('Someone has updated', { parent: parent.constructor.name, }) @@ -228,7 +220,7 @@ export abstract class Indexer { this.logger.info('Updating', { from, to: effect.targetHeight }) try { const newHeight = await this.update(from, effect.targetHeight) - if (Height.gt(newHeight, effect.targetHeight)) { + if (newHeight > effect.targetHeight) { this.logger.critical('Update returned invalid height', { newHeight, max: effect.targetHeight, diff --git a/packages/uif/src/height.ts b/packages/uif/src/height.ts deleted file mode 100644 index 705e53b0..00000000 --- a/packages/uif/src/height.ts +++ /dev/null @@ -1,45 +0,0 @@ -export const Height = { - lt, - lte, - gt, - gte, - min, -} - -function lt(heightA: number | null, heightB: number | null): boolean { - if (heightA === heightB) { - return false - } - return !gt(heightA, heightB) -} - -function lte(heightA: number | null, heightB: number | null): boolean { - return !gt(heightA, heightB) -} - -function gt(heightA: number | null, heightB: number | null): boolean { - if (heightA === null) { - return false - } - if (heightB === null) { - return true - } - return heightA > heightB -} - -function gte(heightA: number | null, heightB: number | null): boolean { - return !lt(heightA, heightB) -} - -function min(...heights: (number | null)[]): number | null { - if (heights.length === 0) { - return null - } - let minHeight = heights[0] ?? null - for (const height of heights) { - if (gt(minHeight, height)) { - minHeight = height - } - } - return minHeight -} diff --git a/packages/uif/src/index.ts b/packages/uif/src/index.ts index 55c38dea..9d6cc6d1 100644 --- a/packages/uif/src/index.ts +++ b/packages/uif/src/index.ts @@ -1,4 +1,3 @@ -export * from './height' export * from './Indexer' export * from './indexers/ChildIndexer' export * from './indexers/multi/MultiIndexer' diff --git a/packages/uif/src/indexers/multi/MultiIndexer.test.ts b/packages/uif/src/indexers/multi/MultiIndexer.test.ts index 39468314..1536a89e 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.test.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.test.ts @@ -123,7 +123,7 @@ describe(MultiIndexer.name, () => { ) await testIndexer.initialize() - const newHeight = await testIndexer.update(null, 500) + const newHeight = await testIndexer.update(0, 500) expect(newHeight).toEqual(99) expect(testIndexer.multiUpdate).not.toHaveBeenCalled() diff --git a/packages/uif/src/indexers/multi/MultiIndexer.ts b/packages/uif/src/indexers/multi/MultiIndexer.ts index 33d0caf0..efaf3668 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.ts @@ -1,6 +1,5 @@ import { Logger } from '@l2beat/backend-tools' -import { Height } from '../../height' import { Indexer, IndexerOptions } from '../../Indexer' import { ChildIndexer } from '../ChildIndexer' import { diffConfigurations } from './diffConfigurations' @@ -93,7 +92,7 @@ export abstract class MultiIndexer extends ChildIndexer { configurations: SavedConfiguration[], ): Promise - async initialize(): Promise { + async initialize(): Promise { const saved = await this.multiInitialize() const { toRemove, toSave, safeHeight } = diffConfigurations( this.configurations, @@ -107,18 +106,10 @@ export abstract class MultiIndexer extends ChildIndexer { return safeHeight } - async update( - currentHeight: number | null, - targetHeight: number, - ): Promise { + async update(currentHeight: number, targetHeight: number): Promise { const range = findRange(this.ranges, currentHeight) - if ( - range.configurations.length === 0 || - // This check is only necessary for TypeScript. If currentHeight is null - // then the first condition will always be true - currentHeight === null - ) { - return Height.min(range.to, targetHeight) + if (range.configurations.length === 0) { + return Math.min(range.to, targetHeight) } const { configurations, minCurrentHeight } = getConfigurationsInRange( @@ -147,7 +138,7 @@ export abstract class MultiIndexer extends ChildIndexer { return newHeight } - async invalidate(targetHeight: number | null): Promise { + async invalidate(targetHeight: number): Promise { return Promise.resolve(targetHeight) } @@ -158,12 +149,10 @@ export abstract class MultiIndexer extends ChildIndexer { function findRange( ranges: ConfigurationRange[], - currentHeight: number | null, + currentHeight: number, ): ConfigurationRange { const range = ranges.find( - (range) => - currentHeight === null || - (range.from <= currentHeight + 1 && range.to > currentHeight), + (range) => range.from <= currentHeight + 1 && range.to > currentHeight, ) if (!range) { throw new Error('Programmer error, there should always be a range') diff --git a/packages/uif/src/indexers/multi/diffConfigurations.ts b/packages/uif/src/indexers/multi/diffConfigurations.ts index c4f47ae5..33940f12 100644 --- a/packages/uif/src/indexers/multi/diffConfigurations.ts +++ b/packages/uif/src/indexers/multi/diffConfigurations.ts @@ -1,4 +1,3 @@ -import { Height } from '../../height' import { Configuration, RemovalConfiguration, @@ -11,9 +10,9 @@ export function diffConfigurations( ): { toRemove: RemovalConfiguration[] toSave: SavedConfiguration[] - safeHeight: number | null + safeHeight: number } { - let safeHeight: number | null = Infinity + let safeHeight = Infinity const knownIds = new Set() const actualMap = new Map(actual.map((c) => [c.id, c])) @@ -42,12 +41,12 @@ export function diffConfigurations( const stored = savedMap.get(c.id) if (!stored) { - safeHeight = Height.min(safeHeight, c.minHeight - 1) + safeHeight = Math.min(safeHeight, c.minHeight - 1) continue } if (stored.minHeight > c.minHeight) { - safeHeight = Height.min(safeHeight, c.minHeight - 1) + safeHeight = Math.min(safeHeight, c.minHeight - 1) // We remove everything because we cannot have gaps in downloaded data // We will re-download everything from the beginning toRemove.push({ @@ -72,11 +71,8 @@ export function diffConfigurations( fromHeightInclusive: c.maxHeight + 1, toHeightInclusive: stored.currentHeight, }) - } else if ( - c.maxHeight === null || - Height.lt(stored.currentHeight, c.maxHeight) - ) { - safeHeight = Height.min(safeHeight, stored.currentHeight) + } else if (c.maxHeight === null || stored.currentHeight < c.maxHeight) { + safeHeight = Math.min(safeHeight, stored.currentHeight) } } diff --git a/packages/uif/src/reducer/handlers/handleParentUpdated.ts b/packages/uif/src/reducer/handlers/handleParentUpdated.ts index d308eb0a..ac02ddba 100644 --- a/packages/uif/src/reducer/handlers/handleParentUpdated.ts +++ b/packages/uif/src/reducer/handlers/handleParentUpdated.ts @@ -1,4 +1,3 @@ -import { Height } from '../../height' import { continueOperations } from '../helpers/continueOperations' import { finishInitialization } from '../helpers/finishInitialization' import { ParentUpdatedAction } from '../types/IndexerAction' @@ -13,8 +12,7 @@ export function handleParentUpdated( ...state, parents: state.parents.map((parent, index) => { if (index === action.index) { - const waiting = - parent.waiting || Height.lt(action.safeHeight, parent.safeHeight) + const waiting = parent.waiting || action.safeHeight < parent.safeHeight return { ...parent, safeHeight: action.safeHeight, diff --git a/packages/uif/src/reducer/handlers/handleUpdateSucceeded.ts b/packages/uif/src/reducer/handlers/handleUpdateSucceeded.ts index 95dbb269..dcd451d3 100644 --- a/packages/uif/src/reducer/handlers/handleUpdateSucceeded.ts +++ b/packages/uif/src/reducer/handlers/handleUpdateSucceeded.ts @@ -1,4 +1,3 @@ -import { Height } from '../../height' import { assertStatus } from '../helpers/assertStatus' import { continueOperations } from '../helpers/continueOperations' import { UpdateSucceededAction } from '../types/IndexerAction' @@ -10,7 +9,7 @@ export function handleUpdateSucceeded( action: UpdateSucceededAction, ): IndexerReducerResult { assertStatus(state.status, 'updating') - if (Height.gte(action.newHeight, state.height)) { + if (action.newHeight >= state.height) { state = { ...state, status: 'idle', @@ -24,10 +23,7 @@ export function handleUpdateSucceeded( state = { ...state, status: 'idle', - invalidateToHeight: Height.min( - action.newHeight, - state.invalidateToHeight, - ), + invalidateToHeight: Math.min(action.newHeight, state.invalidateToHeight), forceInvalidate: true, } } diff --git a/packages/uif/src/reducer/helpers/continueOperations.ts b/packages/uif/src/reducer/helpers/continueOperations.ts index e04955ea..9dc85053 100644 --- a/packages/uif/src/reducer/helpers/continueOperations.ts +++ b/packages/uif/src/reducer/helpers/continueOperations.ts @@ -1,6 +1,5 @@ import assert from 'node:assert' -import { Height } from '../../height' import { IndexerEffect } from '../types/IndexerEffect' import { IndexerReducerResult } from '../types/IndexerReducerResult' import { IndexerState } from '../types/IndexerState' @@ -14,30 +13,25 @@ export function continueOperations( } = {}, ): IndexerReducerResult { const initializedParents = state.parents.filter((x) => x.initialized) - const parentHeight = Height.min( - ...initializedParents.map((x) => x.safeHeight), - ) + const parentHeight = Math.min(...initializedParents.map((x) => x.safeHeight)) if (initializedParents.length > 0) { state = { ...state, - invalidateToHeight: Height.min(state.invalidateToHeight, parentHeight), + invalidateToHeight: Math.min(state.invalidateToHeight, parentHeight), } } const effects: IndexerEffect[] = [] - if ( - Height.lt(state.invalidateToHeight, state.safeHeight) || - options.updateFinished - ) { - const safeHeight = Height.min(state.invalidateToHeight, state.height) + if (state.invalidateToHeight < state.safeHeight || options.updateFinished) { + const safeHeight = Math.min(state.invalidateToHeight, state.height) if (safeHeight !== state.safeHeight) { effects.push({ type: 'SetSafeHeight', safeHeight }) } - if (Height.lt(safeHeight, state.safeHeight)) { + if (safeHeight < state.safeHeight) { state = { ...state, safeHeight, @@ -91,12 +85,11 @@ export function continueOperations( } const shouldInvalidate = - state.forceInvalidate || Height.lt(state.invalidateToHeight, state.height) + state.forceInvalidate || state.invalidateToHeight < state.height const shouldUpdate = !shouldInvalidate && initializedParents.length > 0 && - Height.gt(parentHeight, state.height) && - parentHeight !== null + parentHeight > state.height if (shouldInvalidate) { if (state.invalidateBlocked || state.waiting || state.status !== 'idle') { diff --git a/packages/uif/src/reducer/helpers/finishInitialization.ts b/packages/uif/src/reducer/helpers/finishInitialization.ts index 7e2b10aa..b749de4f 100644 --- a/packages/uif/src/reducer/helpers/finishInitialization.ts +++ b/packages/uif/src/reducer/helpers/finishInitialization.ts @@ -1,4 +1,3 @@ -import { Height } from '../../height' import { IndexerReducerResult } from '../types/IndexerReducerResult' import { IndexerState } from '../types/IndexerState' @@ -22,8 +21,8 @@ export function finishInitialization( } if (state.parents.every((x) => x.initialized)) { - const parentHeight = Height.min(...state.parents.map((x) => x.safeHeight)) - const height = Height.min(parentHeight, state.height) + const parentHeight = Math.min(...state.parents.map((x) => x.safeHeight)) + const height = Math.min(parentHeight, state.height) return [ { diff --git a/packages/uif/src/reducer/types/IndexerAction.ts b/packages/uif/src/reducer/types/IndexerAction.ts index dd39ec54..43f5bd08 100644 --- a/packages/uif/src/reducer/types/IndexerAction.ts +++ b/packages/uif/src/reducer/types/IndexerAction.ts @@ -1,13 +1,13 @@ export interface InitializedAction { type: 'Initialized' - safeHeight: number | null + safeHeight: number childCount: number } export interface ParentUpdatedAction { type: 'ParentUpdated' index: number - safeHeight: number | null + safeHeight: number } export interface ChildReadyAction { @@ -17,8 +17,8 @@ export interface ChildReadyAction { export interface UpdateSucceededAction { type: 'UpdateSucceeded' - from: number | null - newHeight: number | null + from: number + newHeight: number } export interface UpdateFailedAction { @@ -32,7 +32,7 @@ export interface RetryUpdateAction { export interface InvalidateSucceededAction { type: 'InvalidateSucceeded' - targetHeight: number | null + targetHeight: number } export interface InvalidateFailedAction { diff --git a/packages/uif/src/reducer/types/IndexerEffect.ts b/packages/uif/src/reducer/types/IndexerEffect.ts index 8d04c31c..5a7c0d34 100644 --- a/packages/uif/src/reducer/types/IndexerEffect.ts +++ b/packages/uif/src/reducer/types/IndexerEffect.ts @@ -15,12 +15,12 @@ export interface UpdateEffect { export interface InvalidateEffect { type: 'Invalidate' - targetHeight: number | null + targetHeight: number } export interface SetSafeHeightEffect { type: 'SetSafeHeight' - safeHeight: number | null + safeHeight: number } export interface NotifyReadyEffect { diff --git a/packages/uif/src/reducer/types/IndexerState.ts b/packages/uif/src/reducer/types/IndexerState.ts index 12dd5976..5ac64e68 100644 --- a/packages/uif/src/reducer/types/IndexerState.ts +++ b/packages/uif/src/reducer/types/IndexerState.ts @@ -6,12 +6,12 @@ export interface IndexerState { | 'invalidating' | 'ticking' | 'errored' - readonly height: number | null - readonly invalidateToHeight: number | null + readonly height: number + readonly invalidateToHeight: number readonly forceInvalidate: boolean // When we change safe height to a lower value we become waiting // and we mark all children as not ready - readonly safeHeight: number | null + readonly safeHeight: number readonly waiting: boolean readonly tickScheduled: boolean readonly initializedSelf: boolean @@ -19,7 +19,7 @@ export interface IndexerState { readonly initialized: boolean // When the parent changes safeHeight to a lower value // we mark them as waiting and will notify them when we're ready - readonly safeHeight: number | null + readonly safeHeight: number readonly waiting: boolean }[] readonly children: { From 245b3c8d8e9ae60f7fe92a7598923882de04ece5 Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Fri, 22 Mar 2024 14:01:45 +0100 Subject: [PATCH 22/31] Make from inclusive --- .../uif-example/src/blocks/BlockIndexer.ts | 7 +- .../uif-example/src/prices/PriceIndexer.ts | 13 ++-- packages/uif/src/Indexer.ts | 24 +++---- .../src/indexers/multi/MultiIndexer.test.ts | 29 +++++--- .../uif/src/indexers/multi/MultiIndexer.ts | 66 +++++++++---------- 5 files changed, 75 insertions(+), 64 deletions(-) diff --git a/packages/uif-example/src/blocks/BlockIndexer.ts b/packages/uif-example/src/blocks/BlockIndexer.ts index 80ac9410..d0641cf0 100644 --- a/packages/uif-example/src/blocks/BlockIndexer.ts +++ b/packages/uif-example/src/blocks/BlockIndexer.ts @@ -29,14 +29,13 @@ export class BlockIndexer extends ChildIndexer { await this.blockIndexerRepository.saveHeight(height) } - override async update(currentHeight: number): Promise { - const nextHeight = currentHeight + 1 - const timestamp = nextHeight * ONE_HOUR_MS + override async update(from: number): Promise { + const timestamp = from * ONE_HOUR_MS const block = await this.blockService.getBlockNumberBefore(timestamp) await this.blockRepository.save({ number: block, timestamp }) - return nextHeight + return from } override async invalidate(targetHeight: number): Promise { diff --git a/packages/uif-example/src/prices/PriceIndexer.ts b/packages/uif-example/src/prices/PriceIndexer.ts index 467d3026..80a7e98f 100644 --- a/packages/uif-example/src/prices/PriceIndexer.ts +++ b/packages/uif-example/src/prices/PriceIndexer.ts @@ -37,18 +37,17 @@ export class PriceIndexer extends MultiIndexer { } override async multiUpdate( - currentHeight: number, - targetHeight: number, + from: number, + to: number, configurations: UpdateConfiguration[], ): Promise { - const startHour = currentHeight + 1 // we only query 24 hours at a time - const endHour = Math.min(targetHeight, startHour + 23) + const adjustedTo = Math.min(to, from + 23) const prices = await this.priceService.getHourlyPrices( this.apiId, - startHour * ONE_HOUR_MS, - endHour * ONE_HOUR_MS, + from * ONE_HOUR_MS, + adjustedTo * ONE_HOUR_MS, ) const dataToSave = configurations @@ -64,7 +63,7 @@ export class PriceIndexer extends MultiIndexer { }) await this.priceRepository.save(dataToSave) - return endHour + return adjustedTo } override async removeData( diff --git a/packages/uif/src/Indexer.ts b/packages/uif/src/Indexer.ts index 9ef664df..93248688 100644 --- a/packages/uif/src/Indexer.ts +++ b/packages/uif/src/Indexer.ts @@ -68,23 +68,23 @@ export abstract class Indexer { * Implements the main data fetching process. It is up to the indexer to * decide how much data to fetch. For example given `.update(100, 200)`, the * indexer can only fetch data up to 110 and return 110. The next time this - * method will be called with `.update(110, 200)`. + * method will be called with `.update(111, 200)`. * - * @param currentHeight The height that the indexer has synced up to - * previously. This value is exclusive so the indexer should not fetch data - * for this height. + * @param from The height for which the indexer should start syncing data. + * This value is inclusive. * - * @param targetHeight The height that the indexer should sync up to. This value is - * inclusive so the indexer should eventually fetch data for this height. + * @param to The height at which the indexer should end syncing data. This + * value is also inclusive so the indexer should eventually sync data for this + * height. * * @returns The height that the indexer has synced up to. Returning - * `currentHeight` means that the indexer has not synced any data. Returning - * a value greater than `currentHeight` means that the indexer has synced up - * to that height. Returning a value less than `currentHeight` will trigger + * `from` means that the indexer has synced a single data point. Returning + * a value greater than `from` means that the indexer has synced up + * to that height. Returning a value less than `from` will trigger * invalidation down to the returned value. Returning a value greater than - * `targetHeight` is not permitted. + * `to` is not permitted. */ - abstract update(currentHeight: number, targetHeight: number): Promise + abstract update(from: number, to: number): Promise /** * Responsible for invalidating data that was synced previously. It is @@ -216,7 +216,7 @@ export abstract class Indexer { // #region Child methods private async executeUpdate(effect: UpdateEffect): Promise { - const from = this.state.height + const from = this.state.height + 1 this.logger.info('Updating', { from, to: effect.targetHeight }) try { const newHeight = await this.update(from, effect.targetHeight) diff --git a/packages/uif/src/indexers/multi/MultiIndexer.test.ts b/packages/uif/src/indexers/multi/MultiIndexer.test.ts index 1536a89e..3b8ce0a2 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.test.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.test.ts @@ -36,6 +36,19 @@ describe(MultiIndexer.name, () => { expect(testIndexer.removeData).not.toHaveBeenCalled() expect(testIndexer.saveConfigurations).not.toHaveBeenCalled() }) + + it('no synced data', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 400), actual('b', 200, 500)], + [], + ) + + const newHeight = await testIndexer.initialize() + expect(newHeight).toEqual(99) + + expect(testIndexer.removeData).not.toHaveBeenCalled() + expect(testIndexer.saveConfigurations).not.toHaveBeenCalled() + }) }) describe(MultiIndexer.prototype.update.name, () => { @@ -103,10 +116,10 @@ describe(MultiIndexer.name, () => { ) await testIndexer.initialize() - const newHeight = await testIndexer.update(300, 600) + const newHeight = await testIndexer.update(301, 600) expect(newHeight).toEqual(400) - expect(testIndexer.multiUpdate).toHaveBeenOnlyCalledWith(300, 400, [ + expect(testIndexer.multiUpdate).toHaveBeenOnlyCalledWith(301, 400, [ update('a', 100, 400, false), update('b', 200, 500, false), ]) @@ -137,7 +150,7 @@ describe(MultiIndexer.name, () => { ) await testIndexer.initialize() - const newHeight = await testIndexer.update(400, 500) + const newHeight = await testIndexer.update(401, 500) expect(newHeight).toEqual(500) expect(testIndexer.multiUpdate).not.toHaveBeenCalled() @@ -151,7 +164,7 @@ describe(MultiIndexer.name, () => { ) await testIndexer.initialize() - const newHeight = await testIndexer.update(200, 500) + const newHeight = await testIndexer.update(201, 500) expect(newHeight).toEqual(299) expect(testIndexer.multiUpdate).not.toHaveBeenCalled() @@ -208,8 +221,8 @@ describe(MultiIndexer.name, () => { ]) // Next range - expect(await testIndexer.update(200, 500)).toEqual(400) - expect(testIndexer.multiUpdate).toHaveBeenNthCalledWith(3, 200, 400, [ + expect(await testIndexer.update(201, 500)).toEqual(400) + expect(testIndexer.multiUpdate).toHaveBeenNthCalledWith(3, 201, 400, [ update('b', 100, 400, false), ]) expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(3, [ @@ -278,7 +291,7 @@ describe(MultiIndexer.name, () => { testIndexer.multiUpdate.resolvesTo(150) await expect(testIndexer.update(200, 300)).toBeRejectedWith( - /returned height must be between currentHeight and targetHeight/, + /returned height must be between from and to/, ) }) @@ -292,7 +305,7 @@ describe(MultiIndexer.name, () => { testIndexer.multiUpdate.resolvesTo(350) await expect(testIndexer.update(200, 300)).toBeRejectedWith( - /returned height must be between currentHeight and targetHeight/, + /returned height must be between from and to/, ) }) }) diff --git a/packages/uif/src/indexers/multi/MultiIndexer.ts b/packages/uif/src/indexers/multi/MultiIndexer.ts index efaf3668..4b6631fb 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.ts @@ -43,29 +43,29 @@ export abstract class MultiIndexer extends ChildIndexer { * indexer can only fetch data up to 110 and return 110. The next time this * method will be called with `.update(110, 200, [...])`. * - * @param currentHeight The height that the indexer has synced up to - * previously. This value is exclusive so the indexer should not fetch data - * for this height. If the indexer hasn't synced anything previously this - * will equal the minimum height of all configurations - 1. + * @param from The height for which the indexer should start syncing data. + * This value is inclusive. If the indexer hasn't synced anything previously + * this will equal the minimum height of all configurations. * - * @param targetHeight The height that the indexer should sync up to. This value is - * inclusive so the indexer should eventually fetch data for this height. + * @param to The height at which the indexer should end syncing data. This + * value is also inclusive so the indexer should eventually sync data for this + * height. * * @param configurations The configurations that the indexer should use to * sync data. The configurations are guaranteed to be in the range of - * `currentHeight` and `targetHeight`. Some of those configurations might - * have been synced previously for this range. Those configurations - * will include the `hasData` flag set to `true`. + * `from` and `to`. Some of those configurations might have been synced + * previously for this range. Those configurations will include the `hasData` + * flag set to `true`. * * @returns The height that the indexer has synced up to. Returning - * `currentHeight` means that the indexer has not synced any data. Returning - * a value greater than `currentHeight` means that the indexer has synced up - * to that height. Returning a value less than `currentHeight` or greater than - * `targetHeight` is not permitted. + * `from` means that the indexer has synced a single data point. Returning + * a value greater than `from` means that the indexer has synced up + * to that height. Returning a value less than `from` or greater than + * `to` is not permitted. */ abstract multiUpdate( - currentHeight: number, - targetHeight: number, + from: number, + to: number, configurations: UpdateConfiguration[], ): Promise @@ -106,31 +106,32 @@ export abstract class MultiIndexer extends ChildIndexer { return safeHeight } - async update(currentHeight: number, targetHeight: number): Promise { - const range = findRange(this.ranges, currentHeight) + async update(from: number, to: number): Promise { + const range = findRange(this.ranges, from) if (range.configurations.length === 0) { - return Math.min(range.to, targetHeight) + return Math.min(range.to, to) } const { configurations, minCurrentHeight } = getConfigurationsInRange( range, this.saved, - currentHeight, + from, ) - const minTargetHeight = Math.min(range.to, targetHeight, minCurrentHeight) - - const newHeight = await this.multiUpdate( - currentHeight, - minTargetHeight, - configurations, - ) - if (newHeight < currentHeight || newHeight > minTargetHeight) { + const adjustedTo = Math.min(range.to, to, minCurrentHeight) + + this.logger.info('Calling multiUpdate', { + from, + to: adjustedTo, + configurations: configurations.length, + }) + const newHeight = await this.multiUpdate(from, adjustedTo, configurations) + if (newHeight < from || newHeight > adjustedTo) { throw new Error( - 'Programmer error, returned height must be between currentHeight and targetHeight.', + 'Programmer error, returned height must be between from and to (both inclusive).', ) } - if (newHeight > currentHeight) { + if (newHeight > from) { updateSavedConfigurations(this.saved, configurations, newHeight) await this.saveConfigurations(this.saved) } @@ -147,13 +148,12 @@ export abstract class MultiIndexer extends ChildIndexer { } } +// TODO: test this function! function findRange( ranges: ConfigurationRange[], - currentHeight: number, + from: number, ): ConfigurationRange { - const range = ranges.find( - (range) => range.from <= currentHeight + 1 && range.to > currentHeight, - ) + const range = ranges.find((range) => range.from <= from && range.to >= from) if (!range) { throw new Error('Programmer error, there should always be a range') } From b77084fe8c68cafe69f7c2966852390799d40961 Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Fri, 22 Mar 2024 14:50:35 +0100 Subject: [PATCH 23/31] Make everything inclusive and store all configurations --- .../uif-example/src/prices/PriceIndexer.ts | 4 +- .../src/indexers/multi/MultiIndexer.test.ts | 158 ++++++++++++------ .../uif/src/indexers/multi/MultiIndexer.ts | 9 +- .../indexers/multi/diffConfigurations.test.ts | 82 +++++---- .../src/indexers/multi/diffConfigurations.ts | 65 ++++--- .../uif/src/indexers/multi/toRanges.test.ts | 47 +++--- packages/uif/src/indexers/multi/types.ts | 25 ++- 7 files changed, 230 insertions(+), 160 deletions(-) diff --git a/packages/uif-example/src/prices/PriceIndexer.ts b/packages/uif-example/src/prices/PriceIndexer.ts index 80a7e98f..5abc032c 100644 --- a/packages/uif-example/src/prices/PriceIndexer.ts +++ b/packages/uif-example/src/prices/PriceIndexer.ts @@ -72,8 +72,8 @@ export class PriceIndexer extends MultiIndexer { for (const c of configurations) { await this.priceRepository.deletePrices( c.properties.tokenSymbol, - c.fromHeightInclusive, - c.toHeightInclusive, + c.from, + c.to, ) } } diff --git a/packages/uif/src/indexers/multi/MultiIndexer.test.ts b/packages/uif/src/indexers/multi/MultiIndexer.test.ts index 3b8ce0a2..31d7e941 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.test.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.test.ts @@ -2,14 +2,23 @@ import { Logger } from '@l2beat/backend-tools' import { expect, mockFn } from 'earl' import { MultiIndexer } from './MultiIndexer' -import { Configuration, SavedConfiguration } from './types' +import { + Configuration, + RemovalConfiguration, + SavedConfiguration, + UpdateConfiguration, +} from './types' describe(MultiIndexer.name, () => { describe(MultiIndexer.prototype.initialize.name, () => { it('calls multiInitialize and saves configurations', async () => { const testIndexer = new TestMultiIndexer( [actual('a', 100, 400), actual('b', 200, 500)], - [saved('a', 100, 300), saved('b', 200, 300), saved('c', 100, 300)], + [ + saved('a', 100, 400, 300), + saved('b', 200, 500, 300), + saved('c', 100, 300, 300), + ], ) const newHeight = await testIndexer.initialize() @@ -19,27 +28,30 @@ describe(MultiIndexer.name, () => { removal('c', 100, 300), ]) expect(testIndexer.saveConfigurations).toHaveBeenOnlyCalledWith([ - saved('a', 100, 300), - saved('b', 200, 300), + saved('a', 100, 400, 300), + saved('b', 200, 500, 300), ]) }) it('skips calling removeData if there is nothing to remove', async () => { const testIndexer = new TestMultiIndexer( [actual('a', 100, 400), actual('b', 200, 500)], - [saved('a', 100, 400), saved('b', 200, 500)], + [saved('a', 100, 400, 400), saved('b', 200, 500, 500)], ) const newHeight = await testIndexer.initialize() expect(newHeight).toEqual(Infinity) expect(testIndexer.removeData).not.toHaveBeenCalled() - expect(testIndexer.saveConfigurations).not.toHaveBeenCalled() + expect(testIndexer.saveConfigurations).toHaveBeenCalledWith([ + saved('a', 100, 400, 400), + saved('b', 200, 500, 500), + ]) }) it('no synced data', async () => { const testIndexer = new TestMultiIndexer( - [actual('a', 100, 400), actual('b', 200, 500)], + [actual('a', 100, 400), actual('b', 200, null)], [], ) @@ -47,7 +59,33 @@ describe(MultiIndexer.name, () => { expect(newHeight).toEqual(99) expect(testIndexer.removeData).not.toHaveBeenCalled() - expect(testIndexer.saveConfigurations).not.toHaveBeenCalled() + expect(testIndexer.saveConfigurations).toHaveBeenCalledWith([ + saved('a', 100, 400, null), + saved('b', 200, null, null), + ]) + }) + + it('mismatched min and max times', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 500), actual('b', 200, 400), actual('c', 300, null)], + [ + saved('a', 100, 400, 300), + saved('b', 100, 300, 300), + saved('c', 300, 400, 300), + ], + ) + + const newHeight = await testIndexer.initialize() + expect(newHeight).toEqual(300) + + expect(testIndexer.removeData).toHaveBeenCalledWith([ + removal('b', 100, 199), + ]) + expect(testIndexer.saveConfigurations).toHaveBeenCalledWith([ + saved('a', 100, 500, 300), + saved('b', 200, 400, 300), + saved('c', 300, null, 300), + ]) }) }) @@ -65,15 +103,16 @@ describe(MultiIndexer.name, () => { expect(testIndexer.multiUpdate).toHaveBeenOnlyCalledWith(100, 200, [ update('a', 100, 200, false), ]) - expect(testIndexer.saveConfigurations).toHaveBeenOnlyCalledWith([ - saved('a', 100, 200), + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(2, [ + saved('a', 100, 200, 200), + saved('b', 300, 400, null), ]) }) it('calls multiUpdate with a late matching configuration', async () => { const testIndexer = new TestMultiIndexer( [actual('a', 100, 200), actual('b', 300, 400)], - [saved('a', 100, 200)], + [saved('a', 100, 200, 200)], ) await testIndexer.initialize() @@ -83,9 +122,9 @@ describe(MultiIndexer.name, () => { expect(testIndexer.multiUpdate).toHaveBeenOnlyCalledWith(300, 400, [ update('b', 300, 400, false), ]) - expect(testIndexer.saveConfigurations).toHaveBeenOnlyCalledWith([ - saved('a', 100, 200), - saved('b', 300, 400), + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(2, [ + saved('a', 100, 200, 200), + saved('b', 300, 400, 400), ]) }) @@ -103,16 +142,16 @@ describe(MultiIndexer.name, () => { update('a', 100, 200, false), update('b', 100, 400, false), ]) - expect(testIndexer.saveConfigurations).toHaveBeenOnlyCalledWith([ - saved('a', 100, 200), - saved('b', 100, 200), + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(2, [ + saved('a', 100, 200, 200), + saved('b', 100, 400, 200), ]) }) it('calls multiUpdate with two middle matching configurations', async () => { const testIndexer = new TestMultiIndexer( [actual('a', 100, 400), actual('b', 200, 500)], - [saved('a', 100, 300), saved('b', 200, 300)], + [saved('a', 100, 400, 300), saved('b', 200, 500, 300)], ) await testIndexer.initialize() @@ -123,9 +162,9 @@ describe(MultiIndexer.name, () => { update('a', 100, 400, false), update('b', 200, 500, false), ]) - expect(testIndexer.saveConfigurations).toHaveBeenOnlyCalledWith([ - saved('a', 100, 400), - saved('b', 200, 400), + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(2, [ + saved('a', 100, 400, 400), + saved('b', 200, 500, 400), ]) }) @@ -140,7 +179,7 @@ describe(MultiIndexer.name, () => { expect(newHeight).toEqual(99) expect(testIndexer.multiUpdate).not.toHaveBeenCalled() - expect(testIndexer.saveConfigurations).not.toHaveBeenCalled() + expect(testIndexer.saveConfigurations).toHaveBeenCalledTimes(1) }) it('skips calling multiUpdate if we are too late', async () => { @@ -154,7 +193,7 @@ describe(MultiIndexer.name, () => { expect(newHeight).toEqual(500) expect(testIndexer.multiUpdate).not.toHaveBeenCalled() - expect(testIndexer.saveConfigurations).not.toHaveBeenCalled() + expect(testIndexer.saveConfigurations).toHaveBeenCalledTimes(1) }) it('skips calling multiUpdate between configs', async () => { @@ -168,13 +207,13 @@ describe(MultiIndexer.name, () => { expect(newHeight).toEqual(299) expect(testIndexer.multiUpdate).not.toHaveBeenCalled() - expect(testIndexer.saveConfigurations).not.toHaveBeenCalled() + expect(testIndexer.saveConfigurations).toHaveBeenCalledTimes(1) }) it('calls multiUpdate with a matching configuration with data', async () => { const testIndexer = new TestMultiIndexer( [actual('a', 100, 200), actual('b', 100, 400)], - [saved('a', 100, 200)], + [saved('a', 100, 200, 200)], ) await testIndexer.initialize() @@ -185,16 +224,16 @@ describe(MultiIndexer.name, () => { update('a', 100, 200, true), update('b', 100, 400, false), ]) - expect(testIndexer.saveConfigurations).toHaveBeenOnlyCalledWith([ - saved('a', 100, 200), - saved('b', 100, 200), + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(2, [ + saved('a', 100, 200, 200), + saved('b', 100, 400, 200), ]) }) it('multiple update calls', async () => { const testIndexer = new TestMultiIndexer( [actual('a', 100, 200), actual('b', 100, 400)], - [saved('a', 100, 200)], + [saved('a', 100, 200, 200)], ) await testIndexer.initialize() @@ -204,8 +243,8 @@ describe(MultiIndexer.name, () => { update('b', 100, 400, false), ]) expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(1, [ - saved('a', 100, 200), - saved('b', 100, 200), + saved('a', 100, 200, 200), + saved('b', 100, 400, 200), ]) // The same range. In real life might be a result of a parent reorg @@ -216,8 +255,8 @@ describe(MultiIndexer.name, () => { update('b', 100, 400, true), ]) expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(2, [ - saved('a', 100, 200), - saved('b', 100, 200), + saved('a', 100, 200, 200), + saved('b', 100, 400, 200), ]) // Next range @@ -226,8 +265,8 @@ describe(MultiIndexer.name, () => { update('b', 100, 400, false), ]) expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(3, [ - saved('a', 100, 200), - saved('b', 100, 400), + saved('a', 100, 200, 200), + saved('b', 100, 400, 400), ]) }) }) @@ -236,7 +275,7 @@ describe(MultiIndexer.name, () => { it('returns the currentHeight', async () => { const testIndexer = new TestMultiIndexer( [actual('a', 100, 300), actual('b', 100, 400)], - [saved('a', 100, 200), saved('b', 100, 200)], + [saved('a', 100, 300, 200), saved('b', 100, 400, 200)], ) await testIndexer.initialize() @@ -244,13 +283,13 @@ describe(MultiIndexer.name, () => { const newHeight = await testIndexer.update(200, 500) expect(newHeight).toEqual(200) - expect(testIndexer.saveConfigurations).not.toHaveBeenCalled() + expect(testIndexer.saveConfigurations).toHaveBeenCalledTimes(1) }) it('returns the targetHeight', async () => { const testIndexer = new TestMultiIndexer( [actual('a', 100, 300), actual('b', 100, 400)], - [saved('a', 100, 200), saved('b', 100, 200)], + [saved('a', 100, 300, 200), saved('b', 100, 400, 200)], ) await testIndexer.initialize() @@ -258,16 +297,16 @@ describe(MultiIndexer.name, () => { const newHeight = await testIndexer.update(200, 300) expect(newHeight).toEqual(300) - expect(testIndexer.saveConfigurations).toHaveBeenOnlyCalledWith([ - saved('a', 100, 300), - saved('b', 100, 300), + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(2, [ + saved('a', 100, 300, 300), + saved('b', 100, 400, 300), ]) }) it('returns something in between', async () => { const testIndexer = new TestMultiIndexer( [actual('a', 100, 300), actual('b', 100, 400)], - [saved('a', 100, 200), saved('b', 100, 200)], + [saved('a', 100, 300, 200), saved('b', 100, 400, 200)], ) await testIndexer.initialize() @@ -275,16 +314,16 @@ describe(MultiIndexer.name, () => { const newHeight = await testIndexer.update(200, 300) expect(newHeight).toEqual(250) - expect(testIndexer.saveConfigurations).toHaveBeenOnlyCalledWith([ - saved('a', 100, 250), - saved('b', 100, 250), + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(2, [ + saved('a', 100, 300, 250), + saved('b', 100, 400, 250), ]) }) it('cannot return less than currentHeight', async () => { const testIndexer = new TestMultiIndexer( [actual('a', 100, 300), actual('b', 100, 400)], - [saved('a', 100, 200), saved('b', 100, 200)], + [saved('a', 100, 300, 200), saved('b', 100, 400, 200)], ) await testIndexer.initialize() @@ -298,7 +337,7 @@ describe(MultiIndexer.name, () => { it('cannot return more than targetHeight', async () => { const testIndexer = new TestMultiIndexer( [actual('a', 100, 300), actual('b', 100, 400)], - [saved('a', 100, 200), saved('b', 100, 200)], + [saved('a', 100, 300, 200), saved('b', 100, 400, 200)], ) await testIndexer.initialize() @@ -333,12 +372,21 @@ class TestMultiIndexer extends MultiIndexer { mockFn['saveConfigurations']>().resolvesTo(undefined) } -function actual(id: string, minHeight: number, maxHeight: number | null) { +function actual( + id: string, + minHeight: number, + maxHeight: number | null, +): Configuration { return { id, properties: null, minHeight, maxHeight } } -function saved(id: string, minHeight: number, currentHeight: number) { - return { id, properties: null, minHeight, currentHeight } +function saved( + id: string, + minHeight: number, + maxHeight: number | null, + currentHeight: number | null, +): SavedConfiguration { + return { id, properties: null, minHeight, maxHeight, currentHeight } } function update( @@ -346,14 +394,14 @@ function update( minHeight: number, maxHeight: number | null, hasData: boolean, -) { +): UpdateConfiguration { return { id, properties: null, minHeight, maxHeight, hasData } } function removal( id: string, - fromHeightInclusive: number, - toHeightInclusive: number, -) { - return { id, properties: null, fromHeightInclusive, toHeightInclusive } + from: number, + to: number, +): RemovalConfiguration { + return { id, properties: null, from, to } } diff --git a/packages/uif/src/indexers/multi/MultiIndexer.ts b/packages/uif/src/indexers/multi/MultiIndexer.ts index 4b6631fb..e50a7b48 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.ts @@ -101,8 +101,8 @@ export abstract class MultiIndexer extends ChildIndexer { this.saved = toSave if (toRemove.length > 0) { await this.removeData(toRemove) - await this.saveConfigurations(toSave) } + await this.saveConfigurations(toSave) return safeHeight } @@ -169,7 +169,7 @@ function getConfigurationsInRange( const configurations = range.configurations.map( (configuration): UpdateConfiguration => { const saved = savedConfigurations.find((c) => c.id === configuration.id) - if (saved && saved.currentHeight > currentHeight) { + if (saved?.currentHeight != null && saved.currentHeight > currentHeight) { minCurrentHeight = Math.min(minCurrentHeight, saved.currentHeight) return { ...configuration, hasData: true } } else { @@ -192,11 +192,14 @@ function updateSavedConfigurations( id: updated.id, properties: updated.properties, minHeight: updated.minHeight, + maxHeight: updated.maxHeight, currentHeight: newHeight, }) } else { // TODO: test this - saved.currentHeight = Math.max(saved.currentHeight, newHeight) + if (saved.currentHeight === null || saved.currentHeight < newHeight) { + saved.currentHeight = newHeight + } } } } diff --git a/packages/uif/src/indexers/multi/diffConfigurations.test.ts b/packages/uif/src/indexers/multi/diffConfigurations.test.ts index eebfddf0..5e299cae 100644 --- a/packages/uif/src/indexers/multi/diffConfigurations.test.ts +++ b/packages/uif/src/indexers/multi/diffConfigurations.test.ts @@ -1,6 +1,11 @@ import { expect } from 'earl' import { diffConfigurations } from './diffConfigurations' +import { + Configuration, + RemovalConfiguration, + SavedConfiguration, +} from './types' describe(diffConfigurations.name, () => { describe('errors', () => { @@ -28,29 +33,33 @@ describe(diffConfigurations.name, () => { [actual('a', 100, null), actual('b', 200, 300)], [], ) - expect(result).toEqual({ toRemove: [], toSave: [], safeHeight: 99 }) + expect(result).toEqual({ + toRemove: [], + toSave: [saved('a', 100, null, null), saved('b', 200, 300, null)], + safeHeight: 99, + }) }) it('partially synced, both early', () => { const result = diffConfigurations( [actual('a', 100, 400), actual('b', 200, null)], - [saved('a', 100, 300), saved('b', 200, 300)], + [saved('a', 100, 400, 300), saved('b', 200, null, 300)], ) expect(result).toEqual({ toRemove: [], - toSave: [saved('a', 100, 300), saved('b', 200, 300)], + toSave: [saved('a', 100, 400, 300), saved('b', 200, null, 300)], safeHeight: 300, }) }) - it('partially synced, one not yet started', () => { + it('partially synced, one new not yet started', () => { const result = diffConfigurations( [actual('a', 100, 400), actual('b', 555, null)], - [saved('a', 100, 300)], + [saved('a', 100, 400, 300), saved('b', 555, null, null)], ) expect(result).toEqual({ toRemove: [], - toSave: [saved('a', 100, 300)], + toSave: [saved('a', 100, 400, 300), saved('b', 555, null, null)], safeHeight: 300, }) }) @@ -58,11 +67,11 @@ describe(diffConfigurations.name, () => { it('partially synced, one finished', () => { const result = diffConfigurations( [actual('a', 100, 555), actual('b', 200, 300)], - [saved('a', 100, 400), saved('b', 200, 300)], + [saved('a', 100, 555, 400), saved('b', 200, 300, 300)], ) expect(result).toEqual({ toRemove: [], - toSave: [saved('a', 100, 400), saved('b', 200, 300)], + toSave: [saved('a', 100, 555, 400), saved('b', 200, 300, 300)], safeHeight: 400, }) }) @@ -70,11 +79,11 @@ describe(diffConfigurations.name, () => { it('partially synced, one finished, one infinite', () => { const result = diffConfigurations( [actual('a', 100, null), actual('b', 200, 300)], - [saved('a', 100, 400), saved('b', 200, 300)], + [saved('a', 100, null, 400), saved('b', 200, 300, 300)], ) expect(result).toEqual({ toRemove: [], - toSave: [saved('a', 100, 400), saved('b', 200, 300)], + toSave: [saved('a', 100, null, 400), saved('b', 200, 300, 300)], safeHeight: 400, }) }) @@ -82,11 +91,11 @@ describe(diffConfigurations.name, () => { it('both synced', () => { const result = diffConfigurations( [actual('a', 100, 400), actual('b', 200, 300)], - [saved('a', 100, 400), saved('b', 200, 300)], + [saved('a', 100, 400, 400), saved('b', 200, 300, 300)], ) expect(result).toEqual({ toRemove: [], - toSave: [saved('a', 100, 400), saved('b', 200, 300)], + toSave: [saved('a', 100, 400, 400), saved('b', 200, 300, 300)], safeHeight: Infinity, }) }) @@ -96,7 +105,7 @@ describe(diffConfigurations.name, () => { it('empty actual', () => { const result = diffConfigurations( [], - [saved('a', 100, 300), saved('b', 200, 300)], + [saved('a', 100, 400, 300), saved('b', 200, null, 300)], ) expect(result).toEqual({ toRemove: [removal('a', 100, 300), removal('b', 200, 300)], @@ -108,11 +117,11 @@ describe(diffConfigurations.name, () => { it('single removed', () => { const result = diffConfigurations( [actual('b', 200, 400)], - [saved('a', 100, 300), saved('b', 200, 300)], + [saved('a', 100, null, 300), saved('b', 200, 400, 300)], ) expect(result).toEqual({ toRemove: [removal('a', 100, 300)], - toSave: [saved('b', 200, 300)], + toSave: [saved('b', 200, 400, 300)], safeHeight: 300, }) }) @@ -120,11 +129,11 @@ describe(diffConfigurations.name, () => { it('maxHeight updated up', () => { const result = diffConfigurations( [actual('a', 100, 400)], - [saved('a', 100, 300)], + [saved('a', 100, 300, 300)], ) expect(result).toEqual({ toRemove: [], - toSave: [saved('a', 100, 300)], + toSave: [saved('a', 100, 400, 300)], safeHeight: 300, }) }) @@ -132,11 +141,11 @@ describe(diffConfigurations.name, () => { it('maxHeight updated down', () => { const result = diffConfigurations( [actual('a', 100, 200)], - [saved('a', 100, 300)], + [saved('a', 100, 300, 300)], ) expect(result).toEqual({ toRemove: [removal('a', 201, 300)], - toSave: [saved('a', 100, 200)], + toSave: [saved('a', 100, 200, 200)], safeHeight: Infinity, }) }) @@ -144,11 +153,11 @@ describe(diffConfigurations.name, () => { it('minHeight updated up', () => { const result = diffConfigurations( [actual('a', 200, 400)], - [saved('a', 100, 300)], + [saved('a', 100, 400, 300)], ) expect(result).toEqual({ toRemove: [removal('a', 100, 199)], - toSave: [saved('a', 200, 300)], + toSave: [saved('a', 200, 400, 300)], safeHeight: 300, }) }) @@ -156,11 +165,11 @@ describe(diffConfigurations.name, () => { it('minHeight updated down', () => { const result = diffConfigurations( [actual('a', 100, 400)], - [saved('a', 200, 300)], + [saved('a', 200, 400, 300)], ) expect(result).toEqual({ toRemove: [removal('a', 200, 300)], - toSave: [], + toSave: [saved('a', 100, 400, null)], safeHeight: 99, }) }) @@ -168,29 +177,38 @@ describe(diffConfigurations.name, () => { it('both min and max height updated', () => { const result = diffConfigurations( [actual('a', 200, 300)], - [saved('a', 100, 400)], + [saved('a', 100, 400, 400)], ) expect(result).toEqual({ toRemove: [removal('a', 100, 199), removal('a', 301, 400)], - toSave: [saved('a', 200, 300)], + toSave: [saved('a', 200, 300, 300)], safeHeight: Infinity, }) }) }) }) -function actual(id: string, minHeight: number, maxHeight: number | null) { +function actual( + id: string, + minHeight: number, + maxHeight: number | null, +): Configuration { return { id, properties: null, minHeight, maxHeight } } -function saved(id: string, minHeight: number, currentHeight: number) { - return { id, properties: null, minHeight, currentHeight } +function saved( + id: string, + minHeight: number, + maxHeight: number | null, + currentHeight: number | null, +): SavedConfiguration { + return { id, properties: null, minHeight, maxHeight, currentHeight } } function removal( id: string, - fromHeightInclusive: number, - toHeightInclusive: number, -) { - return { id, properties: null, fromHeightInclusive, toHeightInclusive } + from: number, + to: number, +): RemovalConfiguration { + return { id, properties: null, from, to } } diff --git a/packages/uif/src/indexers/multi/diffConfigurations.ts b/packages/uif/src/indexers/multi/diffConfigurations.ts index 33940f12..92bba679 100644 --- a/packages/uif/src/indexers/multi/diffConfigurations.ts +++ b/packages/uif/src/indexers/multi/diffConfigurations.ts @@ -14,19 +14,25 @@ export function diffConfigurations( } { let safeHeight = Infinity - const knownIds = new Set() const actualMap = new Map(actual.map((c) => [c.id, c])) const savedMap = new Map(saved.map((c) => [c.id, c])) - const toRemove: RemovalConfiguration[] = saved - .filter((c) => !actualMap.has(c.id)) - .map((c) => ({ + const toRemove: RemovalConfiguration[] = [] + for (const c of saved) { + if (actualMap.has(c.id) || c.currentHeight === null) { + continue + } + toRemove.push({ id: c.id, properties: c.properties, - fromHeightInclusive: c.minHeight, - toHeightInclusive: c.currentHeight, - })) + from: c.minHeight, + to: c.currentHeight, + }) + } + + const toSave: SavedConfiguration[] = [] + const knownIds = new Set() for (const c of actual) { if (knownIds.has(c.id)) { throw new Error(`Configuration ${c.id} is duplicated!`) @@ -40,8 +46,9 @@ export function diffConfigurations( } const stored = savedMap.get(c.id) - if (!stored) { + if (!stored || stored.currentHeight === null) { safeHeight = Math.min(safeHeight, c.minHeight - 1) + toSave.push({ ...c, currentHeight: null }) continue } @@ -52,15 +59,19 @@ export function diffConfigurations( toRemove.push({ id: stored.id, properties: stored.properties, - fromHeightInclusive: stored.minHeight, - toHeightInclusive: stored.currentHeight, + from: stored.minHeight, + to: stored.currentHeight, }) - } else if (stored.minHeight < c.minHeight) { + toSave.push({ ...c, currentHeight: null }) + continue + } + + if (stored.minHeight < c.minHeight) { toRemove.push({ id: stored.id, properties: stored.properties, - fromHeightInclusive: stored.minHeight, - toHeightInclusive: c.minHeight - 1, + from: stored.minHeight, + to: c.minHeight - 1, }) } @@ -68,31 +79,19 @@ export function diffConfigurations( toRemove.push({ id: stored.id, properties: stored.properties, - fromHeightInclusive: c.maxHeight + 1, - toHeightInclusive: stored.currentHeight, + from: c.maxHeight + 1, + to: stored.currentHeight, }) } else if (c.maxHeight === null || stored.currentHeight < c.maxHeight) { safeHeight = Math.min(safeHeight, stored.currentHeight) } - } - const toSave = saved - .map((c): SavedConfiguration | undefined => { - const actual = actualMap.get(c.id) - if (!actual || actual.minHeight < c.minHeight) { - return undefined - } - return { - id: c.id, - properties: c.properties, - minHeight: actual.minHeight, - currentHeight: - actual.maxHeight === null - ? c.currentHeight - : Math.min(c.currentHeight, actual.maxHeight), - } - }) - .filter((c): c is SavedConfiguration => c !== undefined) + const currentHeight = Math.min( + stored.currentHeight, + c.maxHeight ?? stored.currentHeight, + ) + toSave.push({ ...c, currentHeight }) + } return { toRemove, toSave, safeHeight } } diff --git a/packages/uif/src/indexers/multi/toRanges.test.ts b/packages/uif/src/indexers/multi/toRanges.test.ts index 432bce04..7df05601 100644 --- a/packages/uif/src/indexers/multi/toRanges.test.ts +++ b/packages/uif/src/indexers/multi/toRanges.test.ts @@ -1,6 +1,7 @@ import { expect } from 'earl' import { toRanges } from './toRanges' +import { Configuration } from './types' describe(toRanges.name, () => { it('empty', () => { @@ -11,73 +12,77 @@ describe(toRanges.name, () => { }) it('single infinite configuration', () => { - const ranges = toRanges([config('a', 100, null)]) + const ranges = toRanges([actual('a', 100, null)]) expect(ranges).toEqual([ { from: -Infinity, to: 99, configurations: [] }, - { from: 100, to: Infinity, configurations: [config('a', 100, null)] }, + { from: 100, to: Infinity, configurations: [actual('a', 100, null)] }, ]) }) it('single finite configuration', () => { - const ranges = toRanges([config('a', 100, 300)]) + const ranges = toRanges([actual('a', 100, 300)]) expect(ranges).toEqual([ { from: -Infinity, to: 99, configurations: [] }, - { from: 100, to: 300, configurations: [config('a', 100, 300)] }, + { from: 100, to: 300, configurations: [actual('a', 100, 300)] }, { from: 301, to: Infinity, configurations: [] }, ]) }) it('multiple overlapping configurations', () => { const ranges = toRanges([ - config('a', 100, 300), - config('b', 200, 400), - config('c', 300, 500), + actual('a', 100, 300), + actual('b', 200, 400), + actual('c', 300, 500), ]) expect(ranges).toEqual([ { from: -Infinity, to: 99, configurations: [] }, - { from: 100, to: 199, configurations: [config('a', 100, 300)] }, + { from: 100, to: 199, configurations: [actual('a', 100, 300)] }, { from: 200, to: 299, - configurations: [config('a', 100, 300), config('b', 200, 400)], + configurations: [actual('a', 100, 300), actual('b', 200, 400)], }, { from: 300, to: 300, configurations: [ - config('a', 100, 300), - config('b', 200, 400), - config('c', 300, 500), + actual('a', 100, 300), + actual('b', 200, 400), + actual('c', 300, 500), ], }, { from: 301, to: 400, - configurations: [config('b', 200, 400), config('c', 300, 500)], + configurations: [actual('b', 200, 400), actual('c', 300, 500)], }, - { from: 401, to: 500, configurations: [config('c', 300, 500)] }, + { from: 401, to: 500, configurations: [actual('c', 300, 500)] }, { from: 501, to: Infinity, configurations: [] }, ]) }) it('multiple non-overlapping configurations', () => { const ranges = toRanges([ - config('a', 100, 200), - config('b', 300, 400), - config('c', 500, 600), + actual('a', 100, 200), + actual('b', 300, 400), + actual('c', 500, 600), ]) expect(ranges).toEqual([ { from: -Infinity, to: 99, configurations: [] }, - { from: 100, to: 200, configurations: [config('a', 100, 200)] }, + { from: 100, to: 200, configurations: [actual('a', 100, 200)] }, { from: 201, to: 299, configurations: [] }, - { from: 300, to: 400, configurations: [config('b', 300, 400)] }, + { from: 300, to: 400, configurations: [actual('b', 300, 400)] }, { from: 401, to: 499, configurations: [] }, - { from: 500, to: 600, configurations: [config('c', 500, 600)] }, + { from: 500, to: 600, configurations: [actual('c', 500, 600)] }, { from: 601, to: Infinity, configurations: [] }, ]) }) }) -function config(id: string, minHeight: number, maxHeight: number | null) { +function actual( + id: string, + minHeight: number, + maxHeight: number | null, +): Configuration { return { id, properties: null, minHeight, maxHeight } } diff --git a/packages/uif/src/indexers/multi/types.ts b/packages/uif/src/indexers/multi/types.ts index edd807d3..b2079f5f 100644 --- a/packages/uif/src/indexers/multi/types.ts +++ b/packages/uif/src/indexers/multi/types.ts @@ -1,36 +1,33 @@ export interface Configuration { id: string properties: T + /** Inclusive */ minHeight: number + /** Inclusive */ maxHeight: number | null } -export interface UpdateConfiguration { - id: string - properties: T - minHeight: number - maxHeight: number | null +export interface UpdateConfiguration extends Configuration { hasData: boolean } -export interface SavedConfiguration { - id: string - properties: T - minHeight: number - // TODO: add maxHeight - // TODO: add null, save configurations without syncing - currentHeight: number +export interface SavedConfiguration extends Configuration { + currentHeight: number | null } export interface RemovalConfiguration { id: string properties: T - fromHeightInclusive: number - toHeightInclusive: number + /** Inclusive */ + from: number + /** Inclusive */ + to: number } export interface ConfigurationRange { + /** Inclusive */ from: number + /** Inclusive */ to: number configurations: Configuration[] } From 93c6e4148a12f6cff3c04bff3b4747a1dc272a94 Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Fri, 22 Mar 2024 15:03:16 +0100 Subject: [PATCH 24/31] Add more tests --- .../src/indexers/multi/MultiIndexer.test.ts | 42 +++++++++++++++++-- .../uif/src/indexers/multi/MultiIndexer.ts | 9 ++-- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/packages/uif/src/indexers/multi/MultiIndexer.test.ts b/packages/uif/src/indexers/multi/MultiIndexer.test.ts index 31d7e941..d1820124 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.test.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.test.ts @@ -242,7 +242,7 @@ describe(MultiIndexer.name, () => { update('a', 100, 200, true), update('b', 100, 400, false), ]) - expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(1, [ + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(2, [ saved('a', 100, 200, 200), saved('b', 100, 400, 200), ]) @@ -254,7 +254,7 @@ describe(MultiIndexer.name, () => { update('a', 100, 200, true), update('b', 100, 400, true), ]) - expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(2, [ + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(3, [ saved('a', 100, 200, 200), saved('b', 100, 400, 200), ]) @@ -264,11 +264,47 @@ describe(MultiIndexer.name, () => { expect(testIndexer.multiUpdate).toHaveBeenNthCalledWith(3, 201, 400, [ update('b', 100, 400, false), ]) - expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(3, [ + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(4, [ saved('a', 100, 200, 200), saved('b', 100, 400, 400), ]) }) + + it('correctly updates currentHeight in saved configurations', async () => { + const testIndexer = new TestMultiIndexer( + [actual('a', 100, 500), actual('b', 100, 500), actual('c', 100, 500)], + [ + saved('a', 100, 500, null), + saved('b', 100, 500, 250), + saved('c', 100, 500, 500), + ], + ) + expect(await testIndexer.initialize()).toEqual(99) + + expect(await testIndexer.update(100, 500)).toEqual(250) + expect(testIndexer.multiUpdate).toHaveBeenNthCalledWith(1, 100, 250, [ + update('a', 100, 500, false), + update('b', 100, 500, true), + update('c', 100, 500, true), + ]) + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(2, [ + saved('a', 100, 500, 250), + saved('b', 100, 500, 250), + saved('c', 100, 500, 500), + ]) + + expect(await testIndexer.update(251, 500)).toEqual(500) + expect(testIndexer.multiUpdate).toHaveBeenNthCalledWith(2, 251, 500, [ + update('a', 100, 500, false), + update('b', 100, 500, false), + update('c', 100, 500, true), + ]) + expect(testIndexer.saveConfigurations).toHaveBeenNthCalledWith(3, [ + saved('a', 100, 500, 500), + saved('b', 100, 500, 500), + saved('c', 100, 500, 500), + ]) + }) }) describe('multiUpdate', () => { diff --git a/packages/uif/src/indexers/multi/MultiIndexer.ts b/packages/uif/src/indexers/multi/MultiIndexer.ts index e50a7b48..6d10ad51 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.ts @@ -148,7 +148,6 @@ export abstract class MultiIndexer extends ChildIndexer { } } -// TODO: test this function! function findRange( ranges: ConfigurationRange[], from: number, @@ -169,7 +168,12 @@ function getConfigurationsInRange( const configurations = range.configurations.map( (configuration): UpdateConfiguration => { const saved = savedConfigurations.find((c) => c.id === configuration.id) - if (saved?.currentHeight != null && saved.currentHeight > currentHeight) { + if ( + // eslint-disable-next-line @typescript-eslint/prefer-optional-chain + saved && + saved.currentHeight !== null && + saved.currentHeight > currentHeight + ) { minCurrentHeight = Math.min(minCurrentHeight, saved.currentHeight) return { ...configuration, hasData: true } } else { @@ -196,7 +200,6 @@ function updateSavedConfigurations( currentHeight: newHeight, }) } else { - // TODO: test this if (saved.currentHeight === null || saved.currentHeight < newHeight) { saved.currentHeight = newHeight } From 98b70cfbb4fc370694430102713b245aaae99073 Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Mon, 25 Mar 2024 11:49:19 +0100 Subject: [PATCH 25/31] Add test for update heights --- packages/uif/src/Indexer.test.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/uif/src/Indexer.test.ts b/packages/uif/src/Indexer.test.ts index 49c7419e..c6267e18 100644 --- a/packages/uif/src/Indexer.test.ts +++ b/packages/uif/src/Indexer.test.ts @@ -214,6 +214,23 @@ describe(Indexer.name, () => { clock.uninstall() }) }) + + it('calls update with correct heights', async () => { + const parent = new TestRootIndexer(100) + const child = new TestChildIndexer([parent], 100) + + await parent.start() + await child.start() + await child.finishInvalidate(100) + + await parent.doTick(200) + await parent.finishTick(200) + + expect(child.updateFrom).toEqual(101) // inclusive + expect(child.updateTo).toEqual(200) // inclusive + + await child.finishUpdate(200) + }) }) export async function waitUntil(predicate: () => boolean): Promise { From 53f45fc1ae7d602f834f12d6f0a11397f5786d8a Mon Sep 17 00:00:00 2001 From: antooni Date: Mon, 25 Mar 2024 12:45:41 +0100 Subject: [PATCH 26/31] add more tests --- .../uif/src/indexers/multi/toRanges.test.ts | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/packages/uif/src/indexers/multi/toRanges.test.ts b/packages/uif/src/indexers/multi/toRanges.test.ts index 7df05601..c652e2b7 100644 --- a/packages/uif/src/indexers/multi/toRanges.test.ts +++ b/packages/uif/src/indexers/multi/toRanges.test.ts @@ -77,6 +77,135 @@ describe(toRanges.name, () => { { from: 601, to: Infinity, configurations: [] }, ]) }) + + it('multiple overlapping and non-overlapping configurations', () => { + const ranges = toRanges([ + actual('a', 100, 200), + actual('b', 300, 500), + actual('c', 400, 600), + actual('d', 700, 800), + ]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { from: 100, to: 200, configurations: [actual('a', 100, 200)] }, + { from: 201, to: 299, configurations: [] }, + { from: 300, to: 399, configurations: [actual('b', 300, 500)] }, + { + from: 400, + to: 500, + configurations: [actual('b', 300, 500), actual('c', 400, 600)], + }, + { from: 501, to: 600, configurations: [actual('c', 400, 600)] }, + { from: 601, to: 699, configurations: [] }, + { from: 700, to: 800, configurations: [actual('d', 700, 800)] }, + { from: 801, to: Infinity, configurations: [] }, + ]) + }) + + it('adjacent: one configuration start where other ends', () => { + const ranges = toRanges([actual('a', 100, 200), actual('b', 200, 300)]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { from: 100, to: 199, configurations: [actual('a', 100, 200)] }, + { + from: 200, + to: 200, + configurations: [actual('a', 100, 200), actual('b', 200, 300)], + }, + { + from: 201, + to: 300, + configurations: [actual('b', 200, 300)], + }, + { + from: 301, + to: Infinity, + configurations: [], + }, + ]) + }) + + it('identical: two configurations with exactly the same boundaries', () => { + const ranges = toRanges([actual('a', 100, 200), actual('b', 100, 200)]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { + from: 100, + to: 200, + configurations: [actual('a', 100, 200), actual('b', 100, 200)], + }, + { + from: 201, + to: Infinity, + configurations: [], + }, + ]) + }) + + it('single point: configuration starts and ends in the same time', () => { + const ranges = toRanges([actual('a', 100, 100)]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { + from: 100, + to: 100, + configurations: [actual('a', 100, 100)], + }, + { + from: 101, + to: Infinity, + configurations: [], + }, + ]) + }) + + it('order of inputs does not affect output', () => { + const ranges = toRanges([ + actual('b', 300, 400), + actual('c', 500, 600), + actual('a', 100, 200), + ]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { from: 100, to: 200, configurations: [actual('a', 100, 200)] }, + { from: 201, to: 299, configurations: [] }, + { from: 300, to: 400, configurations: [actual('b', 300, 400)] }, + { from: 401, to: 499, configurations: [] }, + { from: 500, to: 600, configurations: [actual('c', 500, 600)] }, + { from: 601, to: Infinity, configurations: [] }, + ]) + }) + + it('same starting point, multiple maxHeights', () => { + const ranges = toRanges([ + actual('a', 100, 200), + actual('b', 100, 300), + actual('c', 100, 400), + ]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { + from: 100, + to: 200, + configurations: [ + actual('a', 100, 200), + actual('b', 100, 300), + actual('c', 100, 400), + ], + }, + { + from: 201, + to: 300, + configurations: [actual('b', 100, 300), actual('c', 100, 400)], + }, + { + from: 301, + to: 400, + configurations: [actual('c', 100, 400)], + }, + { from: 401, to: Infinity, configurations: [] }, + ]) + }) }) function actual( From aae5dce2b812e7611e16e6cf33473a332dad48b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20T=C3=B3rz?= Date: Mon, 25 Mar 2024 13:44:38 +0100 Subject: [PATCH 27/31] add toRanges test case --- packages/uif/src/indexers/multi/toRanges.test.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/uif/src/indexers/multi/toRanges.test.ts b/packages/uif/src/indexers/multi/toRanges.test.ts index c652e2b7..f27b189f 100644 --- a/packages/uif/src/indexers/multi/toRanges.test.ts +++ b/packages/uif/src/indexers/multi/toRanges.test.ts @@ -28,6 +28,21 @@ describe(toRanges.name, () => { ]) }) + it('multiple overlapping configurations on the edges', () => { + const ranges = toRanges([actual('a', 100, 300), actual('b', 300, 500)]) + expect(ranges).toEqual([ + { from: -Infinity, to: 99, configurations: [] }, + { from: 100, to: 299, configurations: [actual('a', 100, 300)] }, + { + from: 300, + to: 300, + configurations: [actual('a', 100, 300), actual('b', 300, 500)], + }, + { from: 301, to: 500, configurations: [actual('b', 300, 500)] }, + { from: 501, to: Infinity, configurations: [] }, + ]) + }) + it('multiple overlapping configurations', () => { const ranges = toRanges([ actual('a', 100, 300), From 9a7a31ea3966e86054cd79e2ed263fc107ce0cfb Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Mon, 25 Mar 2024 14:01:36 +0100 Subject: [PATCH 28/31] Refactor updateSavedConfigurations to be a method --- .../uif/src/indexers/multi/MultiIndexer.ts | 40 ++++++++----------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/packages/uif/src/indexers/multi/MultiIndexer.ts b/packages/uif/src/indexers/multi/MultiIndexer.ts index 6d10ad51..2342bf32 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.ts @@ -132,13 +132,28 @@ export abstract class MultiIndexer extends ChildIndexer { } if (newHeight > from) { - updateSavedConfigurations(this.saved, configurations, newHeight) + this.updateSavedConfigurations(configurations, newHeight) await this.saveConfigurations(this.saved) } return newHeight } + private updateSavedConfigurations( + updatedConfigurations: UpdateConfiguration[], + newHeight: number, + ): void { + for (const updated of updatedConfigurations) { + const saved = this.saved.find((c) => c.id === updated.id) + if (!saved) { + throw new Error('Programmer error, saved configuration not found') + } + if (saved.currentHeight === null || saved.currentHeight < newHeight) { + saved.currentHeight = newHeight + } + } + } + async invalidate(targetHeight: number): Promise { return Promise.resolve(targetHeight) } @@ -183,26 +198,3 @@ function getConfigurationsInRange( ) return { configurations, minCurrentHeight } } - -function updateSavedConfigurations( - savedConfigurations: SavedConfiguration[], - updatedConfigurations: UpdateConfiguration[], - newHeight: number, -): void { - for (const updated of updatedConfigurations) { - const saved = savedConfigurations.find((c) => c.id === updated.id) - if (!saved) { - savedConfigurations.push({ - id: updated.id, - properties: updated.properties, - minHeight: updated.minHeight, - maxHeight: updated.maxHeight, - currentHeight: newHeight, - }) - } else { - if (saved.currentHeight === null || saved.currentHeight < newHeight) { - saved.currentHeight = newHeight - } - } - } -} From 3e44e1b98a2b0210baf2848ef5ed359045eba58d Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Mon, 25 Mar 2024 14:04:05 +0100 Subject: [PATCH 29/31] Add returns doc comments --- packages/uif/src/Indexer.ts | 3 +++ packages/uif/src/indexers/multi/MultiIndexer.ts | 2 ++ 2 files changed, 5 insertions(+) diff --git a/packages/uif/src/Indexer.ts b/packages/uif/src/Indexer.ts index 93248688..32c9069e 100644 --- a/packages/uif/src/Indexer.ts +++ b/packages/uif/src/Indexer.ts @@ -49,6 +49,9 @@ export abstract class Indexer { * height that was saved previously. If not it can `return this.tick()`. * This method should also schedule a process to request ticks. For example * with `setInterval(() => this.requestTick(), 1000)`. + * + * @returns The height that the indexer has synced up to or the target height + * for the entire system if this is a root indexer. */ abstract initialize(): Promise diff --git a/packages/uif/src/indexers/multi/MultiIndexer.ts b/packages/uif/src/indexers/multi/MultiIndexer.ts index 2342bf32..10935a0e 100644 --- a/packages/uif/src/indexers/multi/MultiIndexer.ts +++ b/packages/uif/src/indexers/multi/MultiIndexer.ts @@ -34,6 +34,8 @@ export abstract class MultiIndexer extends ChildIndexer { * This method is expected to read the configurations that was saved * previously with `setStoredConfigurations`. It shouldn't call * `setStoredConfigurations` itself. + * + * @returns The configurations that were saved previously. */ abstract multiInitialize(): Promise[]> From fb287ad60c14d9c6c2a32f8bd67596ecf0450b1c Mon Sep 17 00:00:00 2001 From: Piotr Szlachciak Date: Mon, 25 Mar 2024 14:07:41 +0100 Subject: [PATCH 30/31] Update uif version --- packages/uif/CHANGELOG.md | 10 ++++++++++ packages/uif/package.json | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/uif/CHANGELOG.md b/packages/uif/CHANGELOG.md index 1eec5ab6..e1d4126b 100644 --- a/packages/uif/CHANGELOG.md +++ b/packages/uif/CHANGELOG.md @@ -1,5 +1,15 @@ # @l2beat/uif +## 0.3.0 + +### Minor Changes + +- Adds `MultiIndexer` +- Changes the update method `from` parameter to be inclusive as opposed to exclusive (which was the previous behavior) +- Renames `getSafeHeight` to `initialize` +- Renames `BaseIndexer` to `Indexer` +- Removes `SliceIndexer` + ## 0.2.4 ### Patch Changes diff --git a/packages/uif/package.json b/packages/uif/package.json index 2c460a71..8f661697 100644 --- a/packages/uif/package.json +++ b/packages/uif/package.json @@ -1,7 +1,7 @@ { "name": "@l2beat/uif", "description": "Universal Indexer Framework.", - "version": "0.2.4", + "version": "0.3.0", "license": "MIT", "repository": "https://github.com/l2beat/tools", "bugs": { From c27807b443391ab381e8f9f908a8144bdf0b07e3 Mon Sep 17 00:00:00 2001 From: antooni Date: Mon, 25 Mar 2024 14:56:23 +0100 Subject: [PATCH 31/31] add test for maxHeight --- .../src/indexers/multi/diffConfigurations.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/uif/src/indexers/multi/diffConfigurations.test.ts b/packages/uif/src/indexers/multi/diffConfigurations.test.ts index 5e299cae..bb745745 100644 --- a/packages/uif/src/indexers/multi/diffConfigurations.test.ts +++ b/packages/uif/src/indexers/multi/diffConfigurations.test.ts @@ -150,6 +150,18 @@ describe(diffConfigurations.name, () => { }) }) + it('maxHeight removed', () => { + const result = diffConfigurations( + [actual('a', 100, null)], + [saved('a', 100, 300, 300)], + ) + expect(result).toEqual({ + toRemove: [], + toSave: [saved('a', 100, null, 300)], + safeHeight: 300, + }) + }) + it('minHeight updated up', () => { const result = diffConfigurations( [actual('a', 200, 400)],