From 33c67c870a1d2b8589aa37f0da1d2e321deecd59 Mon Sep 17 00:00:00 2001 From: rumgrum Date: Sun, 26 May 2024 13:13:15 +0300 Subject: [PATCH 01/15] add wasm adapter --- src/core/config.ts | 7 ++- src/core/env.ts | 5 +- src/core/types.ts | 4 ++ src/data/transformers/index.ts | 4 +- src/data/webhooks/index.ts | 4 +- src/db/models/Contract.ts | 2 +- src/scripts/preCompute.ts | 5 +- src/scripts/revalidate.ts | 9 ++-- src/scripts/transform.ts | 8 +-- src/server/routes/indexer/computer.ts | 6 +-- src/wasmcodes/types.ts | 14 ++++++ src/wasmcodes/wasm-code.adapter.ts | 10 ++++ src/wasmcodes/wasm-code.service.test.ts | 39 ++++++++++++++ src/wasmcodes/wasm-code.service.ts | 67 +++++++++++++++++++++++++ 14 files changed, 161 insertions(+), 23 deletions(-) create mode 100644 src/wasmcodes/types.ts create mode 100644 src/wasmcodes/wasm-code.adapter.ts create mode 100644 src/wasmcodes/wasm-code.service.test.ts create mode 100644 src/wasmcodes/wasm-code.service.ts diff --git a/src/core/config.ts b/src/core/config.ts index e2e5ceb5..1f638b1d 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -1,6 +1,8 @@ import * as fs from 'fs' import path from 'path' +import { WasmCodeService } from '@/wasmcodes/wasm-code.service' + import { Config } from './types' // Constants. @@ -19,6 +21,7 @@ export const loadConfig = (configOverride?: string) => { } config = JSON.parse(fs.readFileSync(configPath, 'utf-8')) + config.wasmCodes = new WasmCodeService(config.codeIds) } return config @@ -27,7 +30,7 @@ export const loadConfig = (configOverride?: string) => { /** * Get code IDs for a list of keys in the config. */ -export const getCodeIdsForKeys = (...keys: string[]) => { +export const getCodeIdsForKeys = (...keys: string[]): number[] => { const config = loadConfig() - return keys.flatMap((key) => config.codeIds?.[key] ?? []) + return config.wasmCodes?.findWasmCodeIdsByKeys(...keys) ?? [] } diff --git a/src/core/env.ts b/src/core/env.ts index 780bbf6a..51f64d18 100644 --- a/src/core/env.ts +++ b/src/core/env.ts @@ -914,10 +914,7 @@ export const getEnv = ({ return } - const codeIdKeys = Object.entries(config.codeIds ?? {}).flatMap( - ([key, value]) => (value?.includes(codeId) ? [key] : []) - ) - return codeIdKeys[0] + return config.wasmCodes?.findWasmCodeKeyById(codeId) } const getSlashEvents: FormulaSlashEventsGetter = async ( diff --git a/src/core/types.ts b/src/core/types.ts index 03c65185..0cc0af7d 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -10,6 +10,7 @@ import { State, WasmTxEvent, } from '@/db' +import { WasmCodeAdapter } from '@/wasmcodes/wasm-code.adapter' type DB = { uri?: string } & Pick< SequelizeOptions, @@ -48,6 +49,9 @@ export type Config = { } // Map some arbitary string to a list of code IDs. codeIds?: Record + + wasmCodes?: WasmCodeAdapter + // If present, sets up Sentry error reporting. sentryDsn?: string // Payment info. diff --git a/src/data/transformers/index.ts b/src/data/transformers/index.ts index 94503237..81a48c7e 100644 --- a/src/data/transformers/index.ts +++ b/src/data/transformers/index.ts @@ -43,8 +43,8 @@ export const getProcessedTransformers = ( ] processedTransformers = _transformers.map(({ filter, ...webhook }) => { - const allCodeIds = filter.codeIdsKeys?.flatMap( - (key) => config.codeIds?.[key] ?? [] + const allCodeIds = config.wasmCodes?.findWasmCodeIdsByKeys( + ...(filter.codeIdsKeys ?? []) ) return { diff --git a/src/data/webhooks/index.ts b/src/data/webhooks/index.ts index aa46f80e..22d02e0f 100644 --- a/src/data/webhooks/index.ts +++ b/src/data/webhooks/index.ts @@ -59,8 +59,8 @@ export const getProcessedWebhooks = ( .filter((webhook): webhook is Webhook => !!webhook) processedWebhooks = _webhooks.map(({ filter, ...webhook }) => { - const allCodeIds = filter.codeIdsKeys?.flatMap( - (key) => config.codeIds?.[key] ?? [] + const allCodeIds = config.wasmCodes?.findWasmCodeIdsByKeys( + ...(filter.codeIdsKeys ?? []) ) return { diff --git a/src/db/models/Contract.ts b/src/db/models/Contract.ts index 98024b97..87efb48f 100644 --- a/src/db/models/Contract.ts +++ b/src/db/models/Contract.ts @@ -54,7 +54,7 @@ export class Contract extends Model { */ matchesCodeIdKeys(...keys: string[]): boolean { const config = loadConfig() - const codeIds = keys.flatMap((key) => config.codeIds?.[key] ?? []) + const codeIds = config.wasmCodes?.findWasmCodeIdsByKeys(...keys) ?? [] return codeIds.includes(this.codeId) } } diff --git a/src/scripts/preCompute.ts b/src/scripts/preCompute.ts index 2c7cc307..81fd617b 100644 --- a/src/scripts/preCompute.ts +++ b/src/scripts/preCompute.ts @@ -94,9 +94,8 @@ export const main = async () => { } else if (options.ids?.length || options.codeIdsKeys?.length) { const codeIds = [ ...(options.ids || []), - ...(options.codeIdsKeys || []).flatMap( - (key: string) => config.codeIds?.[key] - ), + ...(config.wasmCodes?.findWasmCodeIdsByKeys(options.codeIdsKeys || []) ?? + []), ] addresses = ( await Contract.findAll({ diff --git a/src/scripts/revalidate.ts b/src/scripts/revalidate.ts index 4c81b1bc..76376310 100644 --- a/src/scripts/revalidate.ts +++ b/src/scripts/revalidate.ts @@ -59,9 +59,12 @@ const main = async () => { let replaced = 0 const formulasReplaced = new Set() - const codeIds = ( - codeIdsKeys && typeof codeIdsKeys === 'string' ? codeIdsKeys.split(',') : [] - ).flatMap((key) => config.codeIds?.[key] ?? []) + const codeIdsKeysFromStr: string[] = + config.wasmCodes?.extractWasmCodeKeys(codeIdsKeys) ?? [] + + const codeIds = + config.wasmCodes?.findWasmCodeIdsByKeys(...codeIdsKeysFromStr) ?? [] + const contracts = codeIds.length > 0 ? await Contract.findAll({ diff --git a/src/scripts/transform.ts b/src/scripts/transform.ts index 46dd3890..5cd5c2d4 100644 --- a/src/scripts/transform.ts +++ b/src/scripts/transform.ts @@ -74,9 +74,11 @@ const main = async () => { } : {} - const codeIds = ( - codeIdsKeys && typeof codeIdsKeys === 'string' ? codeIdsKeys.split(',') : [] - ).flatMap((key) => config.codeIds?.[key] ?? []) + const codeIdsKeysFromStr = + config.wasmCodes?.extractWasmCodeKeys(codeIdsKeys) ?? [] + const codeIds = + config.wasmCodes?.findWasmCodeIdsByKeys(...codeIdsKeysFromStr) ?? [] + if (typeof codeIdsKeys === 'string' && codeIds.length === 0) { throw new Error('No code IDs found in config') } diff --git a/src/server/routes/indexer/computer.ts b/src/server/routes/indexer/computer.ts index 5dee3278..c15fdc20 100644 --- a/src/server/routes/indexer/computer.ts +++ b/src/server/routes/indexer/computer.ts @@ -294,9 +294,9 @@ export const computer: Router.Middleware = async (ctx) => { let allowed = true if (typedFormula.formula.filter.codeIdsKeys?.length) { - const allCodeIds = typedFormula.formula.filter.codeIdsKeys.flatMap( - (key) => config.codeIds?.[key] ?? [] - ) + const codeIdKeys = typedFormula.formula.filter.codeIdsKeys + const allCodeIds = + config.wasmCodes?.findWasmCodeIdsByKeys(...codeIdKeys) ?? [] allowed &&= allCodeIds.includes(contract.codeId) } diff --git a/src/wasmcodes/types.ts b/src/wasmcodes/types.ts new file mode 100644 index 00000000..f0b36b37 --- /dev/null +++ b/src/wasmcodes/types.ts @@ -0,0 +1,14 @@ +export class WasmCode { + constructor( + private readonly _codeKey: string, + private readonly _codeIds: number[] | undefined + ) {} + + get codeKey(): string { + return this._codeKey + } + + get codeIds(): number[] | undefined { + return this._codeIds + } +} diff --git a/src/wasmcodes/wasm-code.adapter.ts b/src/wasmcodes/wasm-code.adapter.ts new file mode 100644 index 00000000..2de37e39 --- /dev/null +++ b/src/wasmcodes/wasm-code.adapter.ts @@ -0,0 +1,10 @@ +import { WasmCode } from './types' + +export interface WasmCodeAdapter { + getWasmCodes(): WasmCode[] + getWasmCodeAllIds(): number[] + findWasmCodeIdsByKeys(...keys: string[]): number[] + findWasmCodeKeyById(id: number): string | undefined + someWasmKeysHasCodeId(codeId: number | undefined, ...keys: string[]): boolean + extractWasmCodeKeys(input: any): string[] +} diff --git a/src/wasmcodes/wasm-code.service.test.ts b/src/wasmcodes/wasm-code.service.test.ts new file mode 100644 index 00000000..99cfeb82 --- /dev/null +++ b/src/wasmcodes/wasm-code.service.test.ts @@ -0,0 +1,39 @@ +import { WasmCode } from './types' +import { WasmCodeService } from './wasm-code.service' + +test('WasmCodeService', () => { + const codeIds = { + codeKey1: [1, 2, 3], + codeKey2: [4, 5, 6], + } + const wasmCodeService = new WasmCodeService(codeIds) + + expect(wasmCodeService.getWasmCodes()).toEqual([ + new WasmCode('codeKey1', [1, 2, 3]), + new WasmCode('codeKey2', [4, 5, 6]), + ]) + + expect(wasmCodeService.getWasmCodeAllIds()).toEqual([1, 2, 3, 4, 5, 6]) + + expect(wasmCodeService.findWasmCodeIdsByKeys('codeKey1')).toEqual([1, 2, 3]) + expect(wasmCodeService.findWasmCodeIdsByKeys('codeKey2')).toEqual([4, 5, 6]) + expect(wasmCodeService.findWasmCodeIdsByKeys('codeKey1', 'codeKey2')).toEqual( + [1, 2, 3, 4, 5, 6] + ) + + expect(wasmCodeService.findWasmCodeKeyById(1)).toEqual('codeKey1') + expect(wasmCodeService.findWasmCodeKeyById(4)).toEqual('codeKey2') + expect(wasmCodeService.findWasmCodeKeyById(7)).toBeUndefined() + + expect(wasmCodeService.someWasmKeysHasCodeId(1, 'codeKey1')).toBeTruthy() + expect(wasmCodeService.someWasmKeysHasCodeId(4, 'codeKey2')).toBeTruthy() + expect(wasmCodeService.someWasmKeysHasCodeId(7, 'codeKey1')).toBeFalsy() + + expect(wasmCodeService.extractWasmCodeKeys(undefined)).toEqual([]) + expect(wasmCodeService.extractWasmCodeKeys('')).toEqual([]) + expect(wasmCodeService.extractWasmCodeKeys('codeKey1')).toEqual(['codeKey1']) + expect(wasmCodeService.extractWasmCodeKeys('codeKey1,codeKey2')).toEqual([ + 'codeKey1', + 'codeKey2', + ]) +}) diff --git a/src/wasmcodes/wasm-code.service.ts b/src/wasmcodes/wasm-code.service.ts new file mode 100644 index 00000000..6e380fc0 --- /dev/null +++ b/src/wasmcodes/wasm-code.service.ts @@ -0,0 +1,67 @@ +import { WasmCode } from './types' +import { WasmCodeAdapter } from './wasm-code.adapter' + +export class WasmCodeService implements WasmCodeAdapter { + private readonly wasmCodes: WasmCode[] + + constructor(codeIds: Record | undefined) { + if (!codeIds) { + throw new Error('No code IDs provided.') + } + + this.wasmCodes = Object.entries(codeIds).map( + ([codeKey, codeIds]: [string, number[] | undefined]) => + new WasmCode(codeKey, codeIds) + ) + } + + getWasmCodes(): WasmCode[] { + return this.wasmCodes + } + + getWasmCodeAllIds(): number[] { + return this.wasmCodes.flatMap( + (wasmCode: WasmCode) => wasmCode.codeIds ?? [] + ) + } + + findWasmCodeIdsByKeys(...keys: string[]): number[] { + return keys.flatMap( + (key: string) => + this.wasmCodes.find((wasmCode: WasmCode) => wasmCode.codeKey === key) + ?.codeIds ?? [] + ) + } + + findWasmCodeKeyById(codeId: number): string | undefined { + return this.wasmCodes.find((wasmCode: WasmCode) => + wasmCode.codeIds?.includes(codeId) + )?.codeKey + } + + someWasmKeysHasCodeId( + codeId: number | undefined, + ...keys: string[] + ): boolean { + if (!codeId) { + return false + } + return keys.some((key: string) => + this.wasmCodes + .find((wasmCode: WasmCode) => wasmCode.codeKey === key) + ?.codeIds?.includes(codeId) + ) + } + + extractWasmCodeKeys(input: any): string[] { + if (!input) { + return [] + } + + if (typeof input !== 'string') { + return [] + } + + return input.split(',').map((key: string) => key.trim()) + } +} From e7597ac6552dbcc886fd91ddc959ffff33fa1afb Mon Sep 17 00:00:00 2001 From: rumgrum Date: Sun, 26 May 2024 19:28:47 +0300 Subject: [PATCH 02/15] load wasm codes from db --- src/core/config.ts | 10 ++- src/db/connection.ts | 4 + .../20240521012355-create-wasm-code-key.ts | 35 +++++++++ .../20240521022046-create-wasm-code-key-id.ts | 42 +++++++++++ src/db/models/WasmCodeKey.ts | 74 +++++++++++++++++++ src/db/models/WasmCodeKeyId.ts | 44 +++++++++++ src/db/models/index.ts | 4 +- src/scripts/export/process.ts | 6 +- src/wasmcodes/wasm-code.adapter.ts | 3 + src/wasmcodes/wasm-code.service.test.ts | 21 +++++- src/wasmcodes/wasm-code.service.ts | 63 ++++++++++++++-- 11 files changed, 293 insertions(+), 13 deletions(-) create mode 100644 src/db/migrations/20240521012355-create-wasm-code-key.ts create mode 100644 src/db/migrations/20240521022046-create-wasm-code-key-id.ts create mode 100644 src/db/models/WasmCodeKey.ts create mode 100644 src/db/models/WasmCodeKeyId.ts diff --git a/src/core/config.ts b/src/core/config.ts index 1f638b1d..0d058685 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -21,12 +21,20 @@ export const loadConfig = (configOverride?: string) => { } config = JSON.parse(fs.readFileSync(configPath, 'utf-8')) - config.wasmCodes = new WasmCodeService(config.codeIds) } return config } +export const updateConfigWasmCodes = async ( + configToUpdate: Config +): Promise => { + configToUpdate.wasmCodes = await WasmCodeService.newWithWasmCodesFromDB() + configToUpdate.codeIds = configToUpdate.wasmCodes.exportWasmCodes() + config = configToUpdate + return configToUpdate +} + /** * Get code IDs for a list of keys in the config. */ diff --git a/src/db/connection.ts b/src/db/connection.ts index 68a29c7e..98772f5b 100644 --- a/src/db/connection.ts +++ b/src/db/connection.ts @@ -21,6 +21,8 @@ import { StakingSlashEvent, State, Validator, + WasmCodeKey, + WasmCodeKeyId, WasmStateEvent, WasmStateEventTransformation, WasmTxEvent, @@ -47,6 +49,8 @@ const getModelsForType = (type: DbType): SequelizeOptions['models'] => StakingSlashEvent, State, Validator, + WasmCodeKey, + WasmCodeKeyId, WasmStateEvent, WasmStateEventTransformation, WasmTxEvent, diff --git a/src/db/migrations/20240521012355-create-wasm-code-key.ts b/src/db/migrations/20240521012355-create-wasm-code-key.ts new file mode 100644 index 00000000..86c24a6e --- /dev/null +++ b/src/db/migrations/20240521012355-create-wasm-code-key.ts @@ -0,0 +1,35 @@ +import { QueryInterface } from 'sequelize' +import { DataType } from 'sequelize-typescript' + +module.exports = { + async up(queryInterface: QueryInterface) { + await queryInterface.createTable('WasmCodeKeys', { + codeKey: { + primaryKey: true, + allowNull: false, + type: DataType.STRING, + }, + description: { + allowNull: true, + type: DataType.TEXT, + }, + createdAt: { + allowNull: false, + type: DataType.DATE, + defaultValue: new Date(), + }, + updatedAt: { + allowNull: false, + type: DataType.DATE, + defaultValue: new Date(), + }, + }) + await queryInterface.addIndex('WasmCodeKeys', { + unique: true, + fields: ['codeKey'], + }) + }, + async down(queryInterface: QueryInterface) { + await queryInterface.dropTable('WasmCodeKeys') + }, +} diff --git a/src/db/migrations/20240521022046-create-wasm-code-key-id.ts b/src/db/migrations/20240521022046-create-wasm-code-key-id.ts new file mode 100644 index 00000000..f0e0ca44 --- /dev/null +++ b/src/db/migrations/20240521022046-create-wasm-code-key-id.ts @@ -0,0 +1,42 @@ +import { QueryInterface } from 'sequelize' +import { DataType } from 'sequelize-typescript' + +module.exports = { + async up(queryInterface: QueryInterface) { + await queryInterface.createTable('WasmCodeKeyIds', { + codeKey: { + allowNull: false, + type: DataType.STRING, + references: { + model: 'WasmCodeKeys', + key: 'codeKey', + }, + }, + codeKeyId: { + allowNull: false, + type: DataType.INTEGER, + }, + description: { + allowNull: true, + type: DataType.TEXT, + }, + createdAt: { + allowNull: false, + type: DataType.DATE, + defaultValue: new Date(), + }, + updatedAt: { + allowNull: false, + type: DataType.DATE, + defaultValue: new Date(), + }, + }) + await queryInterface.addIndex('WasmCodeKeyIds', { + unique: true, + fields: ['codeKey', 'codeKeyId'], + }) + }, + async down(queryInterface: QueryInterface) { + await queryInterface.dropTable('WasmCodeKeyIds') + }, +} diff --git a/src/db/models/WasmCodeKey.ts b/src/db/models/WasmCodeKey.ts new file mode 100644 index 00000000..82c0f7c3 --- /dev/null +++ b/src/db/models/WasmCodeKey.ts @@ -0,0 +1,74 @@ +import { + AllowNull, + Column, + DataType, + HasMany, + Model, + PrimaryKey, + Table, +} from 'sequelize-typescript' + +import { WasmCodeKeyId } from './WasmCodeKeyId' + +@Table({ + timestamps: true, + indexes: [ + { + unique: true, + fields: ['codeKey'], + }, + ], +}) +export class WasmCodeKey extends Model { + @PrimaryKey + @Column + declare codeKey: string + + @AllowNull(true) + @Column(DataType.TEXT) + declare description: number + + @HasMany(() => WasmCodeKeyId, 'codeKey') + declare codeKeyIds: WasmCodeKeyId[] + + create(params: { + codeKey: string + description: number + }): Promise { + return WasmCodeKey.create(params) + } + + async associateCodeKeyIds(codeKeyIds: WasmCodeKeyId[]): Promise { + await this.$add('codeKeyIds', codeKeyIds) + } + + static async findByKeyIncludeIds( + codeKey: string + ): Promise { + return WasmCodeKey.findOne({ + where: { codeKey }, + include: WasmCodeKeyId, + }) + } + + static async findAllWithIds(): Promise { + return WasmCodeKey.findAll({ + include: WasmCodeKeyId, + }) + } + + static async createWasmCode( + codeKey: string, + codeKeyId: number | number[] + ): Promise { + await WasmCodeKey.upsert({ codeKey }) + const arrayCodeKeyId = Array.isArray(codeKeyId) ? codeKeyId : [codeKeyId] + + await Promise.all( + arrayCodeKeyId.map(async (codeId: number) => { + await WasmCodeKeyId.upsert({ codeKeyId: codeId, codeKey }) + }) + ) + return WasmCodeKey.findByKeyIncludeIds(codeKey) + } +} diff --git a/src/db/models/WasmCodeKeyId.ts b/src/db/models/WasmCodeKeyId.ts new file mode 100644 index 00000000..6f3eb955 --- /dev/null +++ b/src/db/models/WasmCodeKeyId.ts @@ -0,0 +1,44 @@ +import { + AllowNull, + BelongsTo, + Column, + Model, + Table, +} from 'sequelize-typescript' + +import { WasmCodeKey } from './WasmCodeKey' + +@Table({ + timestamps: true, + indexes: [ + { + unique: true, + fields: ['codeKey', 'codeKeyId'], + }, + ], +}) +export class WasmCodeKeyId extends Model { + @AllowNull(false) + @Column + declare codeKey: string + + @AllowNull(false) + @Column + declare codeKeyId: number + + @BelongsTo(() => WasmCodeKey, 'codeKey') + declare codeKeyIds: WasmCodeKey[] + + async create(params: { + codeKey: string + codeKeyId: number + }): Promise { + return WasmCodeKeyId.create(params) + } + + static async findAllWithKeyCode(): Promise { + return WasmCodeKeyId.findAll({ + include: WasmCodeKey, + }) + } +} diff --git a/src/db/models/index.ts b/src/db/models/index.ts index ace0bbe8..22097b8f 100644 --- a/src/db/models/index.ts +++ b/src/db/models/index.ts @@ -3,9 +3,9 @@ export * from './AccountCodeIdSet' export * from './AccountKey' export * from './AccountKeyCredit' export * from './AccountWebhook' +export * from './AccountWebhookCodeIdSet' export * from './AccountWebhookEvent' export * from './AccountWebhookEventAttempt' -export * from './AccountWebhookCodeIdSet' export * from './BankStateEvent' export * from './Computation' export * from './ComputationDependency' @@ -15,6 +15,8 @@ export * from './GovStateEvent' export * from './StakingSlashEvent' export * from './State' export * from './Validator' +export * from './WasmCodeKey' +export * from './WasmCodeKeyId' export * from './WasmStateEvent' export * from './WasmStateEventTransformation' export * from './WasmTxEvent' diff --git a/src/scripts/export/process.ts b/src/scripts/export/process.ts index 3a0f8bbd..31009af1 100644 --- a/src/scripts/export/process.ts +++ b/src/scripts/export/process.ts @@ -1,7 +1,7 @@ import * as Sentry from '@sentry/node' import { Command } from 'commander' -import { DbType, getBullWorker, loadConfig } from '@/core' +import { DbType, getBullWorker, loadConfig, updateConfigWasmCodes } from '@/core' import { State, loadDb } from '@/db' import { workerMakers } from './workers' @@ -26,7 +26,7 @@ program.parse() const { config: _config, update, webhooks } = program.opts() // Load config with config option. -const config = loadConfig(_config) +let config = loadConfig(_config) // Add Sentry error reporting. if (config.sentryDsn) { @@ -44,6 +44,8 @@ const main = async () => { type: DbType.Accounts, }) + config = await updateConfigWasmCodes(config) + // Initialize state. await State.createSingletonIfMissing() diff --git a/src/wasmcodes/wasm-code.adapter.ts b/src/wasmcodes/wasm-code.adapter.ts index 2de37e39..509b893e 100644 --- a/src/wasmcodes/wasm-code.adapter.ts +++ b/src/wasmcodes/wasm-code.adapter.ts @@ -2,9 +2,12 @@ import { WasmCode } from './types' export interface WasmCodeAdapter { getWasmCodes(): WasmCode[] + resetWasmCodes(): void + exportWasmCodes(): Record getWasmCodeAllIds(): number[] findWasmCodeIdsByKeys(...keys: string[]): number[] findWasmCodeKeyById(id: number): string | undefined someWasmKeysHasCodeId(codeId: number | undefined, ...keys: string[]): boolean extractWasmCodeKeys(input: any): string[] + loadWasmCodeIdsFromDB(): Promise } diff --git a/src/wasmcodes/wasm-code.service.test.ts b/src/wasmcodes/wasm-code.service.test.ts index 99cfeb82..ad86af84 100644 --- a/src/wasmcodes/wasm-code.service.test.ts +++ b/src/wasmcodes/wasm-code.service.test.ts @@ -1,7 +1,9 @@ +import { WasmCodeKey } from '@/db/models/WasmCodeKey' + import { WasmCode } from './types' import { WasmCodeService } from './wasm-code.service' -test('WasmCodeService', () => { +test('WasmCodeService', async () => { const codeIds = { codeKey1: [1, 2, 3], codeKey2: [4, 5, 6], @@ -13,6 +15,8 @@ test('WasmCodeService', () => { new WasmCode('codeKey2', [4, 5, 6]), ]) + expect(wasmCodeService.exportWasmCodes()).toEqual(codeIds) + expect(wasmCodeService.getWasmCodeAllIds()).toEqual([1, 2, 3, 4, 5, 6]) expect(wasmCodeService.findWasmCodeIdsByKeys('codeKey1')).toEqual([1, 2, 3]) @@ -36,4 +40,19 @@ test('WasmCodeService', () => { 'codeKey1', 'codeKey2', ]) + + await WasmCodeKey.createWasmCode('codeKey1', 1) + await WasmCodeKey.createWasmCode('codeKey2', [2, 3]) + await WasmCodeKey.createWasmCode('codeKey3', []) + + const wasmCodes = await wasmCodeService.loadWasmCodeIdsFromDB() + + expect(wasmCodes).toEqual([ + new WasmCode('codeKey1', [1]), + new WasmCode('codeKey2', [2, 3]), + new WasmCode('codeKey3', []), + ]) + + const newWasmCodeService = await WasmCodeService.newWithWasmCodesFromDB() + expect(newWasmCodeService.getWasmCodes()).toEqual(wasmCodes) }) diff --git a/src/wasmcodes/wasm-code.service.ts b/src/wasmcodes/wasm-code.service.ts index 6e380fc0..26c21ab1 100644 --- a/src/wasmcodes/wasm-code.service.ts +++ b/src/wasmcodes/wasm-code.service.ts @@ -1,24 +1,57 @@ +import { WasmCodeKey } from '@/db/models/WasmCodeKey' +import { WasmCodeKeyId } from '@/db/models/WasmCodeKeyId' + import { WasmCode } from './types' import { WasmCodeAdapter } from './wasm-code.adapter' export class WasmCodeService implements WasmCodeAdapter { - private readonly wasmCodes: WasmCode[] + private wasmCodes: WasmCode[] = [] - constructor(codeIds: Record | undefined) { - if (!codeIds) { - throw new Error('No code IDs provided.') + constructor( + codeIds: Record | undefined = undefined + ) { + if (codeIds) { + this.addWasmCode( + Object.entries(codeIds).map( + ([codeKey, codeIds]: [string, number[] | undefined]) => + new WasmCode(codeKey, codeIds) + ) + ) } + } - this.wasmCodes = Object.entries(codeIds).map( - ([codeKey, codeIds]: [string, number[] | undefined]) => - new WasmCode(codeKey, codeIds) - ) + static async newWithWasmCodesFromDB(): Promise { + const wasmCodeService = new WasmCodeService() + const wasmCodes = await wasmCodeService.loadWasmCodeIdsFromDB() + wasmCodeService.addWasmCode(wasmCodes) + return wasmCodeService + } + + resetWasmCodes(): void { + this.wasmCodes = [] + } + + addWasmCode(wasmCodes: WasmCode[]): void { + this.wasmCodes.push(...wasmCodes) } getWasmCodes(): WasmCode[] { return this.wasmCodes } + exportWasmCodes(): Record { + return this.wasmCodes.reduce( + ( + acc: Record, + wasmCode: WasmCode + ): Record => ({ + ...acc, + [wasmCode.codeKey]: wasmCode.codeIds, + }), + {} + ) + } + getWasmCodeAllIds(): number[] { return this.wasmCodes.flatMap( (wasmCode: WasmCode) => wasmCode.codeIds ?? [] @@ -64,4 +97,18 @@ export class WasmCodeService implements WasmCodeAdapter { return input.split(',').map((key: string) => key.trim()) } + + async loadWasmCodeIdsFromDB(): Promise { + return WasmCodeKey.findAllWithIds().then((wasmCodeKeys: WasmCodeKey[]) => + wasmCodeKeys.map( + (wasmCodeKey: WasmCodeKey) => + new WasmCode( + wasmCodeKey.codeKey, + wasmCodeKey.codeKeyIds.map( + (wasmCodeKeyId: WasmCodeKeyId) => wasmCodeKeyId.codeKeyId + ) + ) + ) + ) + } } From 694bb7fbf170d83b42491e151b12238e45436d24 Mon Sep 17 00:00:00 2001 From: rumgrum Date: Mon, 27 May 2024 22:41:48 +0300 Subject: [PATCH 03/15] add wasm code updater --- src/core/config.ts | 16 +++++++++---- src/scripts/export/process.ts | 10 +++++--- src/wasmcodes/wasm-code-updater.ts | 31 +++++++++++++++++++++++++ src/wasmcodes/wasm-code.adapter.ts | 1 + src/wasmcodes/wasm-code.service.test.ts | 10 ++++++++ src/wasmcodes/wasm-code.service.ts | 11 +++++++-- 6 files changed, 69 insertions(+), 10 deletions(-) create mode 100644 src/wasmcodes/wasm-code-updater.ts diff --git a/src/core/config.ts b/src/core/config.ts index 0d058685..288c8e81 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -4,6 +4,7 @@ import path from 'path' import { WasmCodeService } from '@/wasmcodes/wasm-code.service' import { Config } from './types' +import { WasmCodeUpdater } from '@/wasmcodes/wasm-code-updater' // Constants. export const CONFIG_FILE = path.join(process.cwd(), './config.json') @@ -27,12 +28,17 @@ export const loadConfig = (configOverride?: string) => { } export const updateConfigWasmCodes = async ( - configToUpdate: Config -): Promise => { - configToUpdate.wasmCodes = await WasmCodeService.newWithWasmCodesFromDB() - configToUpdate.codeIds = configToUpdate.wasmCodes.exportWasmCodes() + configToUpdate: Config, + keepUpdating: boolean = true +) => { + configToUpdate.wasmCodes = config.wasmCodes ?? new WasmCodeService() + await WasmCodeUpdater.getInstance().updateWasmCodes(keepUpdating) config = configToUpdate - return configToUpdate +} + +export const updateConfigCodeIds = async (): Promise => { + await config.wasmCodes?.reloadWasmCodes() + config.codeIds = config.wasmCodes?.exportWasmCodes() } /** diff --git a/src/scripts/export/process.ts b/src/scripts/export/process.ts index 31009af1..839e8174 100644 --- a/src/scripts/export/process.ts +++ b/src/scripts/export/process.ts @@ -1,7 +1,12 @@ import * as Sentry from '@sentry/node' import { Command } from 'commander' -import { DbType, getBullWorker, loadConfig, updateConfigWasmCodes } from '@/core' +import { + DbType, + getBullWorker, + loadConfig, + updateConfigWasmCodes, +} from '@/core' import { State, loadDb } from '@/db' import { workerMakers } from './workers' @@ -44,7 +49,7 @@ const main = async () => { type: DbType.Accounts, }) - config = await updateConfigWasmCodes(config) + await updateConfigWasmCodes(config) // Initialize state. await State.createSingletonIfMissing() @@ -97,7 +102,6 @@ const main = async () => { }) } }) - // Tell pm2 we're ready. if (process.send) { process.send('ready') diff --git a/src/wasmcodes/wasm-code-updater.ts b/src/wasmcodes/wasm-code-updater.ts new file mode 100644 index 00000000..150522de --- /dev/null +++ b/src/wasmcodes/wasm-code-updater.ts @@ -0,0 +1,31 @@ +import { updateConfigCodeIds } from '@/core/config' + +// make it singleton +export class WasmCodeUpdater { + private static instance: WasmCodeUpdater + private interval: NodeJS.Timeout | undefined + static getInstance(): WasmCodeUpdater { + if (!this.instance) { + this.instance = new WasmCodeUpdater() + } + return this.instance + } + private constructor() {} + + async updateWasmCodes(keepUpdating: boolean): Promise { + await updateConfigCodeIds() + + if (keepUpdating) { + if (!this.interval) { + this.interval = setInterval(async () => { + await updateConfigCodeIds() + }, 2000) + } + } else { + if (this.interval) { + clearInterval(this.interval) + this.interval = undefined + } + } + } +} diff --git a/src/wasmcodes/wasm-code.adapter.ts b/src/wasmcodes/wasm-code.adapter.ts index 509b893e..f38b2f78 100644 --- a/src/wasmcodes/wasm-code.adapter.ts +++ b/src/wasmcodes/wasm-code.adapter.ts @@ -10,4 +10,5 @@ export interface WasmCodeAdapter { someWasmKeysHasCodeId(codeId: number | undefined, ...keys: string[]): boolean extractWasmCodeKeys(input: any): string[] loadWasmCodeIdsFromDB(): Promise + reloadWasmCodes(): Promise } diff --git a/src/wasmcodes/wasm-code.service.test.ts b/src/wasmcodes/wasm-code.service.test.ts index ad86af84..9569498a 100644 --- a/src/wasmcodes/wasm-code.service.test.ts +++ b/src/wasmcodes/wasm-code.service.test.ts @@ -55,4 +55,14 @@ test('WasmCodeService', async () => { const newWasmCodeService = await WasmCodeService.newWithWasmCodesFromDB() expect(newWasmCodeService.getWasmCodes()).toEqual(wasmCodes) + + await wasmCodeService.reloadWasmCodes() + expect(wasmCodeService.getWasmCodes()).toEqual(wasmCodes) + + await WasmCodeKey.createWasmCode('codeKey4', []) + await wasmCodeService.reloadWasmCodes() + expect(wasmCodeService.getWasmCodes()).toEqual([ + ...wasmCodes, + new WasmCode('codeKey4', []), + ]) }) diff --git a/src/wasmcodes/wasm-code.service.ts b/src/wasmcodes/wasm-code.service.ts index 26c21ab1..f1c3810e 100644 --- a/src/wasmcodes/wasm-code.service.ts +++ b/src/wasmcodes/wasm-code.service.ts @@ -3,7 +3,6 @@ import { WasmCodeKeyId } from '@/db/models/WasmCodeKeyId' import { WasmCode } from './types' import { WasmCodeAdapter } from './wasm-code.adapter' - export class WasmCodeService implements WasmCodeAdapter { private wasmCodes: WasmCode[] = [] @@ -36,7 +35,9 @@ export class WasmCodeService implements WasmCodeAdapter { } getWasmCodes(): WasmCode[] { - return this.wasmCodes + return this.wasmCodes.sort((a: WasmCode, b: WasmCode) => + a.codeKey.localeCompare(b.codeKey) + ) } exportWasmCodes(): Record { @@ -111,4 +112,10 @@ export class WasmCodeService implements WasmCodeAdapter { ) ) } + + async reloadWasmCodes(): Promise { + const wasmCodes = await this.loadWasmCodeIdsFromDB() + this.resetWasmCodes() + this.addWasmCode(wasmCodes) + } } From 0a6d16d1a70b3c0c538eea6ecdb0923f0b2cf1f2 Mon Sep 17 00:00:00 2001 From: rumgrum Date: Thu, 30 May 2024 02:44:43 +0300 Subject: [PATCH 04/15] update: revision 1 --- src/core/config.ts | 27 ++-- src/core/env.ts | 3 +- src/core/types.ts | 3 - src/data/transformers/index.ts | 3 +- src/data/webhooks/index.ts | 3 +- .../20240521012355-create-wasm-code-key.ts | 6 +- .../20240521022046-create-wasm-code-key-id.ts | 6 +- src/db/models/Contract.ts | 6 +- src/db/models/WasmCodeKey.ts | 9 +- src/db/models/WasmCodeKeyId.ts | 7 - src/scripts/export/process.ts | 2 +- src/scripts/preCompute.ts | 6 +- src/scripts/revalidate.ts | 7 +- src/scripts/transform.ts | 7 +- src/server/routes/indexer/computer.ts | 5 +- src/wasmcodes/wasm-code-updater.ts | 31 ---- src/wasmcodes/wasm-code.adapter.ts | 4 +- src/wasmcodes/wasm-code.service.test.ts | 139 ++++++++++-------- src/wasmcodes/wasm-code.service.ts | 78 +++++----- 19 files changed, 167 insertions(+), 185 deletions(-) delete mode 100644 src/wasmcodes/wasm-code-updater.ts diff --git a/src/core/config.ts b/src/core/config.ts index 288c8e81..e73309be 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -4,7 +4,6 @@ import path from 'path' import { WasmCodeService } from '@/wasmcodes/wasm-code.service' import { Config } from './types' -import { WasmCodeUpdater } from '@/wasmcodes/wasm-code-updater' // Constants. export const CONFIG_FILE = path.join(process.cwd(), './config.json') @@ -27,24 +26,26 @@ export const loadConfig = (configOverride?: string) => { return config } -export const updateConfigWasmCodes = async ( - configToUpdate: Config, - keepUpdating: boolean = true -) => { - configToUpdate.wasmCodes = config.wasmCodes ?? new WasmCodeService() - await WasmCodeUpdater.getInstance().updateWasmCodes(keepUpdating) - config = configToUpdate +export const updateConfigWasmCodes = async (configToUpdate?: Config) => { + const wasmCodeService = await WasmCodeService.newWithWasmCodesFromDB() + updateConfigCodeIds(wasmCodeService.exportWasmCodes()) + + if (configToUpdate) { + configToUpdate.codeIds = config.codeIds + } + + return configToUpdate } -export const updateConfigCodeIds = async (): Promise => { - await config.wasmCodes?.reloadWasmCodes() - config.codeIds = config.wasmCodes?.exportWasmCodes() +export const updateConfigCodeIds = async ( + codeIds: Record +): Promise => { + config.codeIds = codeIds } /** * Get code IDs for a list of keys in the config. */ export const getCodeIdsForKeys = (...keys: string[]): number[] => { - const config = loadConfig() - return config.wasmCodes?.findWasmCodeIdsByKeys(...keys) ?? [] + return WasmCodeService.getInstance().findWasmCodeIdsByKeys(...keys) ?? [] } diff --git a/src/core/env.ts b/src/core/env.ts index 51f64d18..c583cf27 100644 --- a/src/core/env.ts +++ b/src/core/env.ts @@ -11,6 +11,7 @@ import { WasmTxEvent, loadDb, } from '@/db' +import { WasmCodeService } from '@/wasmcodes/wasm-code.service' import { getCodeIdsForKeys, loadConfig } from './config' import { @@ -914,7 +915,7 @@ export const getEnv = ({ return } - return config.wasmCodes?.findWasmCodeKeyById(codeId) + return WasmCodeService.getInstance().findWasmCodeKeyById(codeId)?.[0] } const getSlashEvents: FormulaSlashEventsGetter = async ( diff --git a/src/core/types.ts b/src/core/types.ts index 0cc0af7d..ee6c82c5 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -10,7 +10,6 @@ import { State, WasmTxEvent, } from '@/db' -import { WasmCodeAdapter } from '@/wasmcodes/wasm-code.adapter' type DB = { uri?: string } & Pick< SequelizeOptions, @@ -50,8 +49,6 @@ export type Config = { // Map some arbitary string to a list of code IDs. codeIds?: Record - wasmCodes?: WasmCodeAdapter - // If present, sets up Sentry error reporting. sentryDsn?: string // Payment info. diff --git a/src/data/transformers/index.ts b/src/data/transformers/index.ts index 81a48c7e..122f76d3 100644 --- a/src/data/transformers/index.ts +++ b/src/data/transformers/index.ts @@ -6,6 +6,7 @@ import { Transformer, TransformerMaker, } from '@/core' +import { WasmCodeService } from '@/wasmcodes/wasm-code.service' import common from './common' import dao from './dao' @@ -43,7 +44,7 @@ export const getProcessedTransformers = ( ] processedTransformers = _transformers.map(({ filter, ...webhook }) => { - const allCodeIds = config.wasmCodes?.findWasmCodeIdsByKeys( + const allCodeIds = WasmCodeService.getInstance().findWasmCodeIdsByKeys( ...(filter.codeIdsKeys ?? []) ) diff --git a/src/data/webhooks/index.ts b/src/data/webhooks/index.ts index 22d02e0f..a5697c9c 100644 --- a/src/data/webhooks/index.ts +++ b/src/data/webhooks/index.ts @@ -2,6 +2,7 @@ import * as Sentry from '@sentry/node' import { Config, ProcessedWebhook, Webhook, WebhookMaker } from '@/core' import { State, WasmStateEvent } from '@/db' +import { WasmCodeService } from '@/wasmcodes/wasm-code.service' import { makeProposalCreated } from './discordNotifier' import { makeIndexerCwReceiptPaid } from './indexerCwReceipt' @@ -59,7 +60,7 @@ export const getProcessedWebhooks = ( .filter((webhook): webhook is Webhook => !!webhook) processedWebhooks = _webhooks.map(({ filter, ...webhook }) => { - const allCodeIds = config.wasmCodes?.findWasmCodeIdsByKeys( + const allCodeIds = WasmCodeService.getInstance().findWasmCodeIdsByKeys( ...(filter.codeIdsKeys ?? []) ) diff --git a/src/db/migrations/20240521012355-create-wasm-code-key.ts b/src/db/migrations/20240521012355-create-wasm-code-key.ts index 86c24a6e..c8e5ea56 100644 --- a/src/db/migrations/20240521012355-create-wasm-code-key.ts +++ b/src/db/migrations/20240521012355-create-wasm-code-key.ts @@ -1,4 +1,4 @@ -import { QueryInterface } from 'sequelize' +import { QueryInterface, fn } from 'sequelize' import { DataType } from 'sequelize-typescript' module.exports = { @@ -16,12 +16,12 @@ module.exports = { createdAt: { allowNull: false, type: DataType.DATE, - defaultValue: new Date(), + defaultValue: fn('NOW'), }, updatedAt: { allowNull: false, type: DataType.DATE, - defaultValue: new Date(), + defaultValue: fn('NOW'), }, }) await queryInterface.addIndex('WasmCodeKeys', { diff --git a/src/db/migrations/20240521022046-create-wasm-code-key-id.ts b/src/db/migrations/20240521022046-create-wasm-code-key-id.ts index f0e0ca44..62415d78 100644 --- a/src/db/migrations/20240521022046-create-wasm-code-key-id.ts +++ b/src/db/migrations/20240521022046-create-wasm-code-key-id.ts @@ -1,4 +1,4 @@ -import { QueryInterface } from 'sequelize' +import { QueryInterface, fn } from 'sequelize' import { DataType } from 'sequelize-typescript' module.exports = { @@ -23,12 +23,12 @@ module.exports = { createdAt: { allowNull: false, type: DataType.DATE, - defaultValue: new Date(), + defaultValue: fn('NOW'), }, updatedAt: { allowNull: false, type: DataType.DATE, - defaultValue: new Date(), + defaultValue: fn('NOW'), }, }) await queryInterface.addIndex('WasmCodeKeyIds', { diff --git a/src/db/models/Contract.ts b/src/db/models/Contract.ts index 87efb48f..bd5a0b7b 100644 --- a/src/db/models/Contract.ts +++ b/src/db/models/Contract.ts @@ -7,8 +7,8 @@ import { Table, } from 'sequelize-typescript' -import { loadConfig } from '@/core/config' import { ContractJson } from '@/core/types' +import { WasmCodeService } from '@/wasmcodes/wasm-code.service' @Table({ timestamps: true, @@ -53,8 +53,8 @@ export class Contract extends Model { * from the config. */ matchesCodeIdKeys(...keys: string[]): boolean { - const config = loadConfig() - const codeIds = config.wasmCodes?.findWasmCodeIdsByKeys(...keys) ?? [] + const codeIds = + WasmCodeService.getInstance().findWasmCodeIdsByKeys(...keys) ?? [] return codeIds.includes(this.codeId) } } diff --git a/src/db/models/WasmCodeKey.ts b/src/db/models/WasmCodeKey.ts index 82c0f7c3..cd91a1ae 100644 --- a/src/db/models/WasmCodeKey.ts +++ b/src/db/models/WasmCodeKey.ts @@ -26,18 +26,11 @@ export class WasmCodeKey extends Model { @AllowNull(true) @Column(DataType.TEXT) - declare description: number + declare description: string @HasMany(() => WasmCodeKeyId, 'codeKey') declare codeKeyIds: WasmCodeKeyId[] - create(params: { - codeKey: string - description: number - }): Promise { - return WasmCodeKey.create(params) - } - async associateCodeKeyIds(codeKeyIds: WasmCodeKeyId[]): Promise { await this.$add('codeKeyIds', codeKeyIds) } diff --git a/src/db/models/WasmCodeKeyId.ts b/src/db/models/WasmCodeKeyId.ts index 6f3eb955..adc8641e 100644 --- a/src/db/models/WasmCodeKeyId.ts +++ b/src/db/models/WasmCodeKeyId.ts @@ -29,13 +29,6 @@ export class WasmCodeKeyId extends Model { @BelongsTo(() => WasmCodeKey, 'codeKey') declare codeKeyIds: WasmCodeKey[] - async create(params: { - codeKey: string - codeKeyId: number - }): Promise { - return WasmCodeKeyId.create(params) - } - static async findAllWithKeyCode(): Promise { return WasmCodeKeyId.findAll({ include: WasmCodeKey, diff --git a/src/scripts/export/process.ts b/src/scripts/export/process.ts index 839e8174..7a788283 100644 --- a/src/scripts/export/process.ts +++ b/src/scripts/export/process.ts @@ -31,7 +31,7 @@ program.parse() const { config: _config, update, webhooks } = program.opts() // Load config with config option. -let config = loadConfig(_config) +const config = loadConfig(_config) // Add Sentry error reporting. if (config.sentryDsn) { diff --git a/src/scripts/preCompute.ts b/src/scripts/preCompute.ts index 81fd617b..8ae9679d 100644 --- a/src/scripts/preCompute.ts +++ b/src/scripts/preCompute.ts @@ -12,6 +12,7 @@ import { } from '@/core' import { getTypedFormula } from '@/data' import { Computation, Contract, State, loadDb } from '@/db' +import { WasmCodeService } from '@/wasmcodes/wasm-code.service' export const main = async () => { // Parse arguments. @@ -94,8 +95,9 @@ export const main = async () => { } else if (options.ids?.length || options.codeIdsKeys?.length) { const codeIds = [ ...(options.ids || []), - ...(config.wasmCodes?.findWasmCodeIdsByKeys(options.codeIdsKeys || []) ?? - []), + ...(WasmCodeService.getInstance().findWasmCodeIdsByKeys( + options.codeIdsKeys || [] + ) ?? []), ] addresses = ( await Contract.findAll({ diff --git a/src/scripts/revalidate.ts b/src/scripts/revalidate.ts index 76376310..8a50c3e2 100644 --- a/src/scripts/revalidate.ts +++ b/src/scripts/revalidate.ts @@ -6,6 +6,7 @@ import { Op } from 'sequelize' import { loadConfig } from '@/core' import { Computation, Contract, loadDb } from '@/db' +import { WasmCodeService } from '@/wasmcodes/wasm-code.service' const LOADER_MAP = ['—', '\\', '|', '/'] @@ -60,10 +61,12 @@ const main = async () => { const formulasReplaced = new Set() const codeIdsKeysFromStr: string[] = - config.wasmCodes?.extractWasmCodeKeys(codeIdsKeys) ?? [] + WasmCodeService.getInstance().extractWasmCodeKeys(codeIdsKeys) ?? [] const codeIds = - config.wasmCodes?.findWasmCodeIdsByKeys(...codeIdsKeysFromStr) ?? [] + WasmCodeService.getInstance().findWasmCodeIdsByKeys( + ...codeIdsKeysFromStr + ) ?? [] const contracts = codeIds.length > 0 diff --git a/src/scripts/transform.ts b/src/scripts/transform.ts index 5cd5c2d4..d4805020 100644 --- a/src/scripts/transform.ts +++ b/src/scripts/transform.ts @@ -9,6 +9,7 @@ import { loadDb, updateComputationValidityDependentOnChanges, } from '@/db' +import { WasmCodeService } from '@/wasmcodes/wasm-code.service' const LOADER_MAP = ['—', '\\', '|', '/'] @@ -75,9 +76,11 @@ const main = async () => { : {} const codeIdsKeysFromStr = - config.wasmCodes?.extractWasmCodeKeys(codeIdsKeys) ?? [] + WasmCodeService.getInstance().extractWasmCodeKeys(codeIdsKeys) ?? [] const codeIds = - config.wasmCodes?.findWasmCodeIdsByKeys(...codeIdsKeysFromStr) ?? [] + WasmCodeService.getInstance().findWasmCodeIdsByKeys( + ...codeIdsKeysFromStr + ) ?? [] if (typeof codeIdsKeys === 'string' && codeIds.length === 0) { throw new Error('No code IDs found in config') diff --git a/src/server/routes/indexer/computer.ts b/src/server/routes/indexer/computer.ts index c15fdc20..ec283c0f 100644 --- a/src/server/routes/indexer/computer.ts +++ b/src/server/routes/indexer/computer.ts @@ -22,6 +22,7 @@ import { State, Validator, } from '@/db' +import { WasmCodeService } from '@/wasmcodes/wasm-code.service' import { captureSentryException } from '../../sentry' @@ -296,7 +297,9 @@ export const computer: Router.Middleware = async (ctx) => { if (typedFormula.formula.filter.codeIdsKeys?.length) { const codeIdKeys = typedFormula.formula.filter.codeIdsKeys const allCodeIds = - config.wasmCodes?.findWasmCodeIdsByKeys(...codeIdKeys) ?? [] + WasmCodeService.getInstance().findWasmCodeIdsByKeys( + ...codeIdKeys + ) ?? [] allowed &&= allCodeIds.includes(contract.codeId) } diff --git a/src/wasmcodes/wasm-code-updater.ts b/src/wasmcodes/wasm-code-updater.ts deleted file mode 100644 index 150522de..00000000 --- a/src/wasmcodes/wasm-code-updater.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { updateConfigCodeIds } from '@/core/config' - -// make it singleton -export class WasmCodeUpdater { - private static instance: WasmCodeUpdater - private interval: NodeJS.Timeout | undefined - static getInstance(): WasmCodeUpdater { - if (!this.instance) { - this.instance = new WasmCodeUpdater() - } - return this.instance - } - private constructor() {} - - async updateWasmCodes(keepUpdating: boolean): Promise { - await updateConfigCodeIds() - - if (keepUpdating) { - if (!this.interval) { - this.interval = setInterval(async () => { - await updateConfigCodeIds() - }, 2000) - } - } else { - if (this.interval) { - clearInterval(this.interval) - this.interval = undefined - } - } - } -} diff --git a/src/wasmcodes/wasm-code.adapter.ts b/src/wasmcodes/wasm-code.adapter.ts index f38b2f78..b0d28d6b 100644 --- a/src/wasmcodes/wasm-code.adapter.ts +++ b/src/wasmcodes/wasm-code.adapter.ts @@ -4,10 +4,8 @@ export interface WasmCodeAdapter { getWasmCodes(): WasmCode[] resetWasmCodes(): void exportWasmCodes(): Record - getWasmCodeAllIds(): number[] findWasmCodeIdsByKeys(...keys: string[]): number[] - findWasmCodeKeyById(id: number): string | undefined - someWasmKeysHasCodeId(codeId: number | undefined, ...keys: string[]): boolean + findWasmCodeKeyById(id: number): string[] | undefined extractWasmCodeKeys(input: any): string[] loadWasmCodeIdsFromDB(): Promise reloadWasmCodes(): Promise diff --git a/src/wasmcodes/wasm-code.service.test.ts b/src/wasmcodes/wasm-code.service.test.ts index 9569498a..09b2fe22 100644 --- a/src/wasmcodes/wasm-code.service.test.ts +++ b/src/wasmcodes/wasm-code.service.test.ts @@ -1,68 +1,83 @@ +import { WasmCodeKeyId } from '@/db' import { WasmCodeKey } from '@/db/models/WasmCodeKey' import { WasmCode } from './types' import { WasmCodeService } from './wasm-code.service' -test('WasmCodeService', async () => { - const codeIds = { - codeKey1: [1, 2, 3], - codeKey2: [4, 5, 6], - } - const wasmCodeService = new WasmCodeService(codeIds) - - expect(wasmCodeService.getWasmCodes()).toEqual([ - new WasmCode('codeKey1', [1, 2, 3]), - new WasmCode('codeKey2', [4, 5, 6]), - ]) - - expect(wasmCodeService.exportWasmCodes()).toEqual(codeIds) - - expect(wasmCodeService.getWasmCodeAllIds()).toEqual([1, 2, 3, 4, 5, 6]) - - expect(wasmCodeService.findWasmCodeIdsByKeys('codeKey1')).toEqual([1, 2, 3]) - expect(wasmCodeService.findWasmCodeIdsByKeys('codeKey2')).toEqual([4, 5, 6]) - expect(wasmCodeService.findWasmCodeIdsByKeys('codeKey1', 'codeKey2')).toEqual( - [1, 2, 3, 4, 5, 6] - ) - - expect(wasmCodeService.findWasmCodeKeyById(1)).toEqual('codeKey1') - expect(wasmCodeService.findWasmCodeKeyById(4)).toEqual('codeKey2') - expect(wasmCodeService.findWasmCodeKeyById(7)).toBeUndefined() - - expect(wasmCodeService.someWasmKeysHasCodeId(1, 'codeKey1')).toBeTruthy() - expect(wasmCodeService.someWasmKeysHasCodeId(4, 'codeKey2')).toBeTruthy() - expect(wasmCodeService.someWasmKeysHasCodeId(7, 'codeKey1')).toBeFalsy() - - expect(wasmCodeService.extractWasmCodeKeys(undefined)).toEqual([]) - expect(wasmCodeService.extractWasmCodeKeys('')).toEqual([]) - expect(wasmCodeService.extractWasmCodeKeys('codeKey1')).toEqual(['codeKey1']) - expect(wasmCodeService.extractWasmCodeKeys('codeKey1,codeKey2')).toEqual([ - 'codeKey1', - 'codeKey2', - ]) - - await WasmCodeKey.createWasmCode('codeKey1', 1) - await WasmCodeKey.createWasmCode('codeKey2', [2, 3]) - await WasmCodeKey.createWasmCode('codeKey3', []) - - const wasmCodes = await wasmCodeService.loadWasmCodeIdsFromDB() - - expect(wasmCodes).toEqual([ - new WasmCode('codeKey1', [1]), - new WasmCode('codeKey2', [2, 3]), - new WasmCode('codeKey3', []), - ]) - - const newWasmCodeService = await WasmCodeService.newWithWasmCodesFromDB() - expect(newWasmCodeService.getWasmCodes()).toEqual(wasmCodes) - - await wasmCodeService.reloadWasmCodes() - expect(wasmCodeService.getWasmCodes()).toEqual(wasmCodes) - - await WasmCodeKey.createWasmCode('codeKey4', []) - await wasmCodeService.reloadWasmCodes() - expect(wasmCodeService.getWasmCodes()).toEqual([ - ...wasmCodes, - new WasmCode('codeKey4', []), - ]) +describe('WasmCodeService tests', () => { + let wasmCodeService: WasmCodeService + + afterAll(async () => { + await wasmCodeService.stopUpdateWasmCodes() + }) + + test('WasmCodeService', async () => { + const codeIds = { + codeKey1: [1, 2, 3], + codeKey2: [4, 5, 6], + codeKey3: [1, 3, 5], + } + + await WasmCodeKey.createWasmCode('codeKey1', [1, 2, 3]) + await WasmCodeKey.createWasmCode('codeKey2', [4, 5, 6]) + await WasmCodeKey.createWasmCode('codeKey3', [1, 3, 5]) + + wasmCodeService = await WasmCodeService.newWithWasmCodesFromDB() + + expect(wasmCodeService.getWasmCodes()).toEqual([ + new WasmCode('codeKey1', [1, 2, 3]), + new WasmCode('codeKey2', [4, 5, 6]), + new WasmCode('codeKey3', [1, 3, 5]), + ]) + + expect(wasmCodeService.exportWasmCodes()).toEqual(codeIds) + + expect(wasmCodeService.findWasmCodeIdsByKeys('codeKey1')).toEqual([1, 2, 3]) + expect(wasmCodeService.findWasmCodeIdsByKeys('codeKey2')).toEqual([4, 5, 6]) + expect( + wasmCodeService.findWasmCodeIdsByKeys('codeKey1', 'codeKey2') + ).toEqual([1, 2, 3, 4, 5, 6]) + + expect(wasmCodeService.findWasmCodeKeyById(1)).toEqual([ + 'codeKey1', + 'codeKey3', + ]) + expect(wasmCodeService.findWasmCodeKeyById(4)).toEqual(['codeKey2']) + expect(wasmCodeService.findWasmCodeKeyById(7)).toEqual([]) + + expect(wasmCodeService.extractWasmCodeKeys(undefined)).toEqual([]) + expect(wasmCodeService.extractWasmCodeKeys('')).toEqual([]) + expect(wasmCodeService.extractWasmCodeKeys('codeKey1')).toEqual([ + 'codeKey1', + ]) + expect(wasmCodeService.extractWasmCodeKeys('codeKey1,codeKey2')).toEqual([ + 'codeKey1', + 'codeKey2', + ]) + + await WasmCodeKeyId.destroy({ where: {} }) + await WasmCodeKey.destroy({ where: {} }) + + await WasmCodeKey.createWasmCode('codeKey1', 1) + await WasmCodeKey.createWasmCode('codeKey2', [2, 3]) + await WasmCodeKey.createWasmCode('codeKey3', []) + + const wasmCodes = await wasmCodeService.loadWasmCodeIdsFromDB() + + expect(wasmCodes).toEqual([ + new WasmCode('codeKey1', [1]), + new WasmCode('codeKey2', [2, 3]), + new WasmCode('codeKey3', []), + ]) + + await wasmCodeService.reloadWasmCodes() + expect(wasmCodeService.getWasmCodes()).toEqual(wasmCodes) + + await WasmCodeKey.createWasmCode('codeKey4', []) + await wasmCodeService.reloadWasmCodes() + expect(wasmCodeService.getWasmCodes()).toEqual([ + ...wasmCodes, + new WasmCode('codeKey4', []), + ]) + }) }) diff --git a/src/wasmcodes/wasm-code.service.ts b/src/wasmcodes/wasm-code.service.ts index f1c3810e..70a4ad0e 100644 --- a/src/wasmcodes/wasm-code.service.ts +++ b/src/wasmcodes/wasm-code.service.ts @@ -1,3 +1,4 @@ +import { updateConfigCodeIds } from '@/core/config' import { WasmCodeKey } from '@/db/models/WasmCodeKey' import { WasmCodeKeyId } from '@/db/models/WasmCodeKeyId' @@ -5,25 +6,30 @@ import { WasmCode } from './types' import { WasmCodeAdapter } from './wasm-code.adapter' export class WasmCodeService implements WasmCodeAdapter { private wasmCodes: WasmCode[] = [] + private interval: NodeJS.Timeout | undefined - constructor( - codeIds: Record | undefined = undefined - ) { - if (codeIds) { - this.addWasmCode( - Object.entries(codeIds).map( - ([codeKey, codeIds]: [string, number[] | undefined]) => - new WasmCode(codeKey, codeIds) - ) - ) + static instance: WasmCodeService + + private constructor() {} + + static getInstance(): WasmCodeService { + if (!this.instance) { + throw new Error('WasmCodeService not initialized') } + return this.instance } static async newWithWasmCodesFromDB(): Promise { + if (this.instance) { + return this.getInstance() + } + const wasmCodeService = new WasmCodeService() const wasmCodes = await wasmCodeService.loadWasmCodeIdsFromDB() wasmCodeService.addWasmCode(wasmCodes) - return wasmCodeService + await wasmCodeService.updateWasmCodes() + this.instance = wasmCodeService + return this.getInstance() } resetWasmCodes(): void { @@ -35,9 +41,12 @@ export class WasmCodeService implements WasmCodeAdapter { } getWasmCodes(): WasmCode[] { - return this.wasmCodes.sort((a: WasmCode, b: WasmCode) => - a.codeKey.localeCompare(b.codeKey) - ) + return this.wasmCodes + .sort((a: WasmCode, b: WasmCode) => a.codeKey.localeCompare(b.codeKey)) + .map( + (wasmCode: WasmCode) => + new WasmCode(wasmCode.codeKey, wasmCode.codeIds?.sort()) + ) } exportWasmCodes(): Record { @@ -53,12 +62,6 @@ export class WasmCodeService implements WasmCodeAdapter { ) } - getWasmCodeAllIds(): number[] { - return this.wasmCodes.flatMap( - (wasmCode: WasmCode) => wasmCode.codeIds ?? [] - ) - } - findWasmCodeIdsByKeys(...keys: string[]): number[] { return keys.flatMap( (key: string) => @@ -67,24 +70,10 @@ export class WasmCodeService implements WasmCodeAdapter { ) } - findWasmCodeKeyById(codeId: number): string | undefined { - return this.wasmCodes.find((wasmCode: WasmCode) => - wasmCode.codeIds?.includes(codeId) - )?.codeKey - } - - someWasmKeysHasCodeId( - codeId: number | undefined, - ...keys: string[] - ): boolean { - if (!codeId) { - return false - } - return keys.some((key: string) => - this.wasmCodes - .find((wasmCode: WasmCode) => wasmCode.codeKey === key) - ?.codeIds?.includes(codeId) - ) + findWasmCodeKeyById(codeId: number): string[] | undefined { + return this.wasmCodes + .filter((wasmCode: WasmCode) => wasmCode.codeIds?.includes(codeId)) + .map((wasmCode: WasmCode) => wasmCode.codeKey) } extractWasmCodeKeys(input: any): string[] { @@ -118,4 +107,17 @@ export class WasmCodeService implements WasmCodeAdapter { this.resetWasmCodes() this.addWasmCode(wasmCodes) } + + async updateWasmCodes(): Promise { + this.interval = setInterval(async () => { + await this.reloadWasmCodes() + await updateConfigCodeIds(this.exportWasmCodes()) + }, 2000) + } + + async stopUpdateWasmCodes(): Promise { + if (this.interval) { + clearInterval(this.interval) + } + } } From cbbfaab0e2476c3a4656a26e54f06a980e3ccb17 Mon Sep 17 00:00:00 2001 From: rumgrum Date: Thu, 30 May 2024 02:56:49 +0300 Subject: [PATCH 05/15] move wasm-codes to src/services directory --- src/core/config.ts | 2 +- src/core/env.ts | 2 +- src/data/transformers/index.ts | 2 +- src/data/webhooks/index.ts | 2 +- src/db/models/Contract.ts | 2 +- src/scripts/preCompute.ts | 2 +- src/scripts/revalidate.ts | 2 +- src/scripts/transform.ts | 2 +- src/server/routes/indexer/computer.ts | 2 +- src/services/wasm-codes/index.ts | 3 +++ src/{wasmcodes => services/wasm-codes}/types.ts | 0 src/{wasmcodes => services/wasm-codes}/wasm-code.adapter.ts | 0 .../wasm-codes}/wasm-code.service.test.ts | 0 src/{wasmcodes => services/wasm-codes}/wasm-code.service.ts | 0 14 files changed, 12 insertions(+), 9 deletions(-) create mode 100644 src/services/wasm-codes/index.ts rename src/{wasmcodes => services/wasm-codes}/types.ts (100%) rename src/{wasmcodes => services/wasm-codes}/wasm-code.adapter.ts (100%) rename src/{wasmcodes => services/wasm-codes}/wasm-code.service.test.ts (100%) rename src/{wasmcodes => services/wasm-codes}/wasm-code.service.ts (100%) diff --git a/src/core/config.ts b/src/core/config.ts index e73309be..0700b1e5 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -1,7 +1,7 @@ import * as fs from 'fs' import path from 'path' -import { WasmCodeService } from '@/wasmcodes/wasm-code.service' +import { WasmCodeService } from '@/services/wasm-codes' import { Config } from './types' diff --git a/src/core/env.ts b/src/core/env.ts index c583cf27..8bc8edaa 100644 --- a/src/core/env.ts +++ b/src/core/env.ts @@ -11,7 +11,7 @@ import { WasmTxEvent, loadDb, } from '@/db' -import { WasmCodeService } from '@/wasmcodes/wasm-code.service' +import { WasmCodeService } from '@/services/wasm-codes' import { getCodeIdsForKeys, loadConfig } from './config' import { diff --git a/src/data/transformers/index.ts b/src/data/transformers/index.ts index 122f76d3..a70df1b7 100644 --- a/src/data/transformers/index.ts +++ b/src/data/transformers/index.ts @@ -6,7 +6,7 @@ import { Transformer, TransformerMaker, } from '@/core' -import { WasmCodeService } from '@/wasmcodes/wasm-code.service' +import { WasmCodeService } from '@/services/wasm-codes' import common from './common' import dao from './dao' diff --git a/src/data/webhooks/index.ts b/src/data/webhooks/index.ts index a5697c9c..bd1681fb 100644 --- a/src/data/webhooks/index.ts +++ b/src/data/webhooks/index.ts @@ -2,7 +2,7 @@ import * as Sentry from '@sentry/node' import { Config, ProcessedWebhook, Webhook, WebhookMaker } from '@/core' import { State, WasmStateEvent } from '@/db' -import { WasmCodeService } from '@/wasmcodes/wasm-code.service' +import { WasmCodeService } from '@/services/wasm-codes' import { makeProposalCreated } from './discordNotifier' import { makeIndexerCwReceiptPaid } from './indexerCwReceipt' diff --git a/src/db/models/Contract.ts b/src/db/models/Contract.ts index bd5a0b7b..a41e1009 100644 --- a/src/db/models/Contract.ts +++ b/src/db/models/Contract.ts @@ -8,7 +8,7 @@ import { } from 'sequelize-typescript' import { ContractJson } from '@/core/types' -import { WasmCodeService } from '@/wasmcodes/wasm-code.service' +import { WasmCodeService } from '@/services/wasm-codes' @Table({ timestamps: true, diff --git a/src/scripts/preCompute.ts b/src/scripts/preCompute.ts index 8ae9679d..69733d05 100644 --- a/src/scripts/preCompute.ts +++ b/src/scripts/preCompute.ts @@ -12,7 +12,7 @@ import { } from '@/core' import { getTypedFormula } from '@/data' import { Computation, Contract, State, loadDb } from '@/db' -import { WasmCodeService } from '@/wasmcodes/wasm-code.service' +import { WasmCodeService } from '@/services/wasm-codes' export const main = async () => { // Parse arguments. diff --git a/src/scripts/revalidate.ts b/src/scripts/revalidate.ts index 8a50c3e2..836d3a2e 100644 --- a/src/scripts/revalidate.ts +++ b/src/scripts/revalidate.ts @@ -6,7 +6,7 @@ import { Op } from 'sequelize' import { loadConfig } from '@/core' import { Computation, Contract, loadDb } from '@/db' -import { WasmCodeService } from '@/wasmcodes/wasm-code.service' +import { WasmCodeService } from '@/services/wasm-codes' const LOADER_MAP = ['—', '\\', '|', '/'] diff --git a/src/scripts/transform.ts b/src/scripts/transform.ts index d4805020..6602b098 100644 --- a/src/scripts/transform.ts +++ b/src/scripts/transform.ts @@ -9,7 +9,7 @@ import { loadDb, updateComputationValidityDependentOnChanges, } from '@/db' -import { WasmCodeService } from '@/wasmcodes/wasm-code.service' +import { WasmCodeService } from '@/services/wasm-codes' const LOADER_MAP = ['—', '\\', '|', '/'] diff --git a/src/server/routes/indexer/computer.ts b/src/server/routes/indexer/computer.ts index ec283c0f..2c443cca 100644 --- a/src/server/routes/indexer/computer.ts +++ b/src/server/routes/indexer/computer.ts @@ -22,7 +22,7 @@ import { State, Validator, } from '@/db' -import { WasmCodeService } from '@/wasmcodes/wasm-code.service' +import { WasmCodeService } from '@/services/wasm-codes' import { captureSentryException } from '../../sentry' diff --git a/src/services/wasm-codes/index.ts b/src/services/wasm-codes/index.ts new file mode 100644 index 00000000..4de6ccbb --- /dev/null +++ b/src/services/wasm-codes/index.ts @@ -0,0 +1,3 @@ +export * from './wasm-code.adapter' +export * from './wasm-code.service' +export * from './types' diff --git a/src/wasmcodes/types.ts b/src/services/wasm-codes/types.ts similarity index 100% rename from src/wasmcodes/types.ts rename to src/services/wasm-codes/types.ts diff --git a/src/wasmcodes/wasm-code.adapter.ts b/src/services/wasm-codes/wasm-code.adapter.ts similarity index 100% rename from src/wasmcodes/wasm-code.adapter.ts rename to src/services/wasm-codes/wasm-code.adapter.ts diff --git a/src/wasmcodes/wasm-code.service.test.ts b/src/services/wasm-codes/wasm-code.service.test.ts similarity index 100% rename from src/wasmcodes/wasm-code.service.test.ts rename to src/services/wasm-codes/wasm-code.service.test.ts diff --git a/src/wasmcodes/wasm-code.service.ts b/src/services/wasm-codes/wasm-code.service.ts similarity index 100% rename from src/wasmcodes/wasm-code.service.ts rename to src/services/wasm-codes/wasm-code.service.ts From c4b2a688e66b11efff0dd01034cf16004822d5cf Mon Sep 17 00:00:00 2001 From: rumgrum Date: Sat, 8 Jun 2024 10:02:57 +0300 Subject: [PATCH 06/15] refactor: revision 2 --- src/core/env.ts | 2 +- src/services/wasm-codes/wasm-code.adapter.ts | 5 +- .../wasm-codes/wasm-code.service.test.ts | 33 ++++++---- src/services/wasm-codes/wasm-code.service.ts | 64 +++++++++---------- 4 files changed, 54 insertions(+), 50 deletions(-) diff --git a/src/core/env.ts b/src/core/env.ts index 8bc8edaa..08d08db7 100644 --- a/src/core/env.ts +++ b/src/core/env.ts @@ -915,7 +915,7 @@ export const getEnv = ({ return } - return WasmCodeService.getInstance().findWasmCodeKeyById(codeId)?.[0] + return WasmCodeService.getInstance().findWasmCodeKeysById(codeId)[0] } const getSlashEvents: FormulaSlashEventsGetter = async ( diff --git a/src/services/wasm-codes/wasm-code.adapter.ts b/src/services/wasm-codes/wasm-code.adapter.ts index b0d28d6b..49dd15d8 100644 --- a/src/services/wasm-codes/wasm-code.adapter.ts +++ b/src/services/wasm-codes/wasm-code.adapter.ts @@ -5,8 +5,7 @@ export interface WasmCodeAdapter { resetWasmCodes(): void exportWasmCodes(): Record findWasmCodeIdsByKeys(...keys: string[]): number[] - findWasmCodeKeyById(id: number): string[] | undefined - extractWasmCodeKeys(input: any): string[] - loadWasmCodeIdsFromDB(): Promise + findWasmCodeKeysById(id: number): string[] | undefined + loadWasmCodeIdsFromDB(): Promise reloadWasmCodes(): Promise } diff --git a/src/services/wasm-codes/wasm-code.service.test.ts b/src/services/wasm-codes/wasm-code.service.test.ts index 09b2fe22..b453086f 100644 --- a/src/services/wasm-codes/wasm-code.service.test.ts +++ b/src/services/wasm-codes/wasm-code.service.test.ts @@ -7,8 +7,8 @@ import { WasmCodeService } from './wasm-code.service' describe('WasmCodeService tests', () => { let wasmCodeService: WasmCodeService - afterAll(async () => { - await wasmCodeService.stopUpdateWasmCodes() + afterAll(() => { + wasmCodeService.stopUpdater() }) test('WasmCodeService', async () => { @@ -38,37 +38,42 @@ describe('WasmCodeService tests', () => { wasmCodeService.findWasmCodeIdsByKeys('codeKey1', 'codeKey2') ).toEqual([1, 2, 3, 4, 5, 6]) - expect(wasmCodeService.findWasmCodeKeyById(1)).toEqual([ + expect(wasmCodeService.findWasmCodeKeysById(1)).toEqual([ 'codeKey1', 'codeKey3', ]) - expect(wasmCodeService.findWasmCodeKeyById(4)).toEqual(['codeKey2']) - expect(wasmCodeService.findWasmCodeKeyById(7)).toEqual([]) + expect(wasmCodeService.findWasmCodeKeysById(4)).toEqual(['codeKey2']) + expect(wasmCodeService.findWasmCodeKeysById(7)).toEqual([]) - expect(wasmCodeService.extractWasmCodeKeys(undefined)).toEqual([]) - expect(wasmCodeService.extractWasmCodeKeys('')).toEqual([]) - expect(wasmCodeService.extractWasmCodeKeys('codeKey1')).toEqual([ + expect(WasmCodeService.extractWasmCodeKeys(undefined)).toEqual([]) + expect(WasmCodeService.extractWasmCodeKeys('')).toEqual([]) + expect(WasmCodeService.extractWasmCodeKeys('codeKey1')).toEqual([ 'codeKey1', ]) - expect(wasmCodeService.extractWasmCodeKeys('codeKey1,codeKey2')).toEqual([ + expect(WasmCodeService.extractWasmCodeKeys('codeKey1,codeKey2')).toEqual([ 'codeKey1', 'codeKey2', ]) - await WasmCodeKeyId.destroy({ where: {} }) - await WasmCodeKey.destroy({ where: {} }) + await WasmCodeKeyId.truncate() + await WasmCodeKey.truncate({ + cascade: true, + }) await WasmCodeKey.createWasmCode('codeKey1', 1) await WasmCodeKey.createWasmCode('codeKey2', [2, 3]) await WasmCodeKey.createWasmCode('codeKey3', []) - const wasmCodes = await wasmCodeService.loadWasmCodeIdsFromDB() + wasmCodeService.resetWasmCodes() + await wasmCodeService.loadWasmCodeIdsFromDB() - expect(wasmCodes).toEqual([ + const wasmCodes = [ new WasmCode('codeKey1', [1]), new WasmCode('codeKey2', [2, 3]), new WasmCode('codeKey3', []), - ]) + ] + + expect(wasmCodeService.getWasmCodes()).toEqual(wasmCodes) await wasmCodeService.reloadWasmCodes() expect(wasmCodeService.getWasmCodes()).toEqual(wasmCodes) diff --git a/src/services/wasm-codes/wasm-code.service.ts b/src/services/wasm-codes/wasm-code.service.ts index 70a4ad0e..ad4275c0 100644 --- a/src/services/wasm-codes/wasm-code.service.ts +++ b/src/services/wasm-codes/wasm-code.service.ts @@ -14,22 +14,16 @@ export class WasmCodeService implements WasmCodeAdapter { static getInstance(): WasmCodeService { if (!this.instance) { - throw new Error('WasmCodeService not initialized') + this.instance = new WasmCodeService() } return this.instance } static async newWithWasmCodesFromDB(): Promise { - if (this.instance) { - return this.getInstance() - } - - const wasmCodeService = new WasmCodeService() - const wasmCodes = await wasmCodeService.loadWasmCodeIdsFromDB() - wasmCodeService.addWasmCode(wasmCodes) - await wasmCodeService.updateWasmCodes() - this.instance = wasmCodeService - return this.getInstance() + const wasmCodeService = WasmCodeService.getInstance() + await wasmCodeService.loadWasmCodeIdsFromDB() + wasmCodeService.startUpdater() + return wasmCodeService } resetWasmCodes(): void { @@ -42,11 +36,11 @@ export class WasmCodeService implements WasmCodeAdapter { getWasmCodes(): WasmCode[] { return this.wasmCodes + .map((wasmCode: WasmCode) => { + wasmCode.codeIds?.sort() + return wasmCode + }) .sort((a: WasmCode, b: WasmCode) => a.codeKey.localeCompare(b.codeKey)) - .map( - (wasmCode: WasmCode) => - new WasmCode(wasmCode.codeKey, wasmCode.codeIds?.sort()) - ) } exportWasmCodes(): Record { @@ -56,7 +50,7 @@ export class WasmCodeService implements WasmCodeAdapter { wasmCode: WasmCode ): Record => ({ ...acc, - [wasmCode.codeKey]: wasmCode.codeIds, + [wasmCode.codeKey]: wasmCode.codeIds?.sort(), }), {} ) @@ -70,13 +64,13 @@ export class WasmCodeService implements WasmCodeAdapter { ) } - findWasmCodeKeyById(codeId: number): string[] | undefined { + findWasmCodeKeysById(codeId: number): string[] { return this.wasmCodes .filter((wasmCode: WasmCode) => wasmCode.codeIds?.includes(codeId)) .map((wasmCode: WasmCode) => wasmCode.codeKey) } - extractWasmCodeKeys(input: any): string[] { + static extractWasmCodeKeys(input: any): string[] { if (!input) { return [] } @@ -88,36 +82,42 @@ export class WasmCodeService implements WasmCodeAdapter { return input.split(',').map((key: string) => key.trim()) } - async loadWasmCodeIdsFromDB(): Promise { - return WasmCodeKey.findAllWithIds().then((wasmCodeKeys: WasmCodeKey[]) => - wasmCodeKeys.map( - (wasmCodeKey: WasmCodeKey) => - new WasmCode( - wasmCodeKey.codeKey, - wasmCodeKey.codeKeyIds.map( - (wasmCodeKeyId: WasmCodeKeyId) => wasmCodeKeyId.codeKeyId - ) + async loadWasmCodeIdsFromDB(): Promise { + const wasmCodesFromDB = await WasmCodeKey.findAllWithIds() + + const wasmCodes = wasmCodesFromDB.map( + (wasmCodeKey: WasmCodeKey) => + new WasmCode( + wasmCodeKey.codeKey, + wasmCodeKey.codeKeyIds.map( + (wasmCodeKeyId: WasmCodeKeyId) => wasmCodeKeyId.codeKeyId ) - ) + ) ) + + this.addWasmCode(wasmCodes) } async reloadWasmCodes(): Promise { - const wasmCodes = await this.loadWasmCodeIdsFromDB() this.resetWasmCodes() - this.addWasmCode(wasmCodes) + await this.loadWasmCodeIdsFromDB() } - async updateWasmCodes(): Promise { + startUpdater(): void { + if (this.interval) { + return + } + this.interval = setInterval(async () => { await this.reloadWasmCodes() await updateConfigCodeIds(this.exportWasmCodes()) }, 2000) } - async stopUpdateWasmCodes(): Promise { + stopUpdater(): void { if (this.interval) { clearInterval(this.interval) + this.interval = undefined } } } From e4e0e73c7944b73d79c895ab338be8a1229ff3a4 Mon Sep 17 00:00:00 2001 From: rumgrum Date: Sat, 8 Jun 2024 10:14:59 +0300 Subject: [PATCH 07/15] fix(findWasmCodeKeysById): return type --- src/services/wasm-codes/wasm-code.adapter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/wasm-codes/wasm-code.adapter.ts b/src/services/wasm-codes/wasm-code.adapter.ts index 49dd15d8..6015020d 100644 --- a/src/services/wasm-codes/wasm-code.adapter.ts +++ b/src/services/wasm-codes/wasm-code.adapter.ts @@ -5,7 +5,7 @@ export interface WasmCodeAdapter { resetWasmCodes(): void exportWasmCodes(): Record findWasmCodeIdsByKeys(...keys: string[]): number[] - findWasmCodeKeysById(id: number): string[] | undefined + findWasmCodeKeysById(id: number): string[] loadWasmCodeIdsFromDB(): Promise reloadWasmCodes(): Promise } From 636c4679bd12a99ca2086fee991578de48add202 Mon Sep 17 00:00:00 2001 From: rumgrum Date: Tue, 11 Jun 2024 06:58:52 +0300 Subject: [PATCH 08/15] refactor: revision 3 --- src/core/config.ts | 2 +- src/services/wasm-codes/wasm-code.adapter.ts | 3 +-- .../wasm-codes/wasm-code.service.test.ts | 9 +++---- src/services/wasm-codes/wasm-code.service.ts | 25 ++++++++++--------- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/core/config.ts b/src/core/config.ts index 0700b1e5..38ba7be0 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -27,7 +27,7 @@ export const loadConfig = (configOverride?: string) => { } export const updateConfigWasmCodes = async (configToUpdate?: Config) => { - const wasmCodeService = await WasmCodeService.newWithWasmCodesFromDB() + const wasmCodeService = await WasmCodeService.setUpInstance() updateConfigCodeIds(wasmCodeService.exportWasmCodes()) if (configToUpdate) { diff --git a/src/services/wasm-codes/wasm-code.adapter.ts b/src/services/wasm-codes/wasm-code.adapter.ts index 6015020d..1cb5b01a 100644 --- a/src/services/wasm-codes/wasm-code.adapter.ts +++ b/src/services/wasm-codes/wasm-code.adapter.ts @@ -6,6 +6,5 @@ export interface WasmCodeAdapter { exportWasmCodes(): Record findWasmCodeIdsByKeys(...keys: string[]): number[] findWasmCodeKeysById(id: number): string[] - loadWasmCodeIdsFromDB(): Promise - reloadWasmCodes(): Promise + reloadWasmCodeIdsFromDB(): Promise } diff --git a/src/services/wasm-codes/wasm-code.service.test.ts b/src/services/wasm-codes/wasm-code.service.test.ts index b453086f..78ae84aa 100644 --- a/src/services/wasm-codes/wasm-code.service.test.ts +++ b/src/services/wasm-codes/wasm-code.service.test.ts @@ -22,7 +22,7 @@ describe('WasmCodeService tests', () => { await WasmCodeKey.createWasmCode('codeKey2', [4, 5, 6]) await WasmCodeKey.createWasmCode('codeKey3', [1, 3, 5]) - wasmCodeService = await WasmCodeService.newWithWasmCodesFromDB() + wasmCodeService = await WasmCodeService.setUpInstance() expect(wasmCodeService.getWasmCodes()).toEqual([ new WasmCode('codeKey1', [1, 2, 3]), @@ -64,8 +64,7 @@ describe('WasmCodeService tests', () => { await WasmCodeKey.createWasmCode('codeKey2', [2, 3]) await WasmCodeKey.createWasmCode('codeKey3', []) - wasmCodeService.resetWasmCodes() - await wasmCodeService.loadWasmCodeIdsFromDB() + await wasmCodeService.reloadWasmCodeIdsFromDB() const wasmCodes = [ new WasmCode('codeKey1', [1]), @@ -75,11 +74,11 @@ describe('WasmCodeService tests', () => { expect(wasmCodeService.getWasmCodes()).toEqual(wasmCodes) - await wasmCodeService.reloadWasmCodes() + await wasmCodeService.reloadWasmCodeIdsFromDB() expect(wasmCodeService.getWasmCodes()).toEqual(wasmCodes) await WasmCodeKey.createWasmCode('codeKey4', []) - await wasmCodeService.reloadWasmCodes() + await wasmCodeService.reloadWasmCodeIdsFromDB() expect(wasmCodeService.getWasmCodes()).toEqual([ ...wasmCodes, new WasmCode('codeKey4', []), diff --git a/src/services/wasm-codes/wasm-code.service.ts b/src/services/wasm-codes/wasm-code.service.ts index ad4275c0..0fad4565 100644 --- a/src/services/wasm-codes/wasm-code.service.ts +++ b/src/services/wasm-codes/wasm-code.service.ts @@ -14,14 +14,19 @@ export class WasmCodeService implements WasmCodeAdapter { static getInstance(): WasmCodeService { if (!this.instance) { - this.instance = new WasmCodeService() + throw new Error( + 'WasmCodeService not initialized because WasmCodeService.setUpInstance was never called' + ) } return this.instance } - static async newWithWasmCodesFromDB(): Promise { - const wasmCodeService = WasmCodeService.getInstance() - await wasmCodeService.loadWasmCodeIdsFromDB() + static async setUpInstance(): Promise { + if (this.instance) { + return this.instance + } + const wasmCodeService = new WasmCodeService() + await wasmCodeService.reloadWasmCodeIdsFromDB() wasmCodeService.startUpdater() return wasmCodeService } @@ -30,7 +35,7 @@ export class WasmCodeService implements WasmCodeAdapter { this.wasmCodes = [] } - addWasmCode(wasmCodes: WasmCode[]): void { + addWasmCodes(wasmCodes: WasmCode[]): void { this.wasmCodes.push(...wasmCodes) } @@ -82,7 +87,7 @@ export class WasmCodeService implements WasmCodeAdapter { return input.split(',').map((key: string) => key.trim()) } - async loadWasmCodeIdsFromDB(): Promise { + async reloadWasmCodeIdsFromDB(): Promise { const wasmCodesFromDB = await WasmCodeKey.findAllWithIds() const wasmCodes = wasmCodesFromDB.map( @@ -95,12 +100,8 @@ export class WasmCodeService implements WasmCodeAdapter { ) ) - this.addWasmCode(wasmCodes) - } - - async reloadWasmCodes(): Promise { this.resetWasmCodes() - await this.loadWasmCodeIdsFromDB() + this.addWasmCodes(wasmCodes) } startUpdater(): void { @@ -109,7 +110,7 @@ export class WasmCodeService implements WasmCodeAdapter { } this.interval = setInterval(async () => { - await this.reloadWasmCodes() + await this.reloadWasmCodeIdsFromDB() await updateConfigCodeIds(this.exportWasmCodes()) }, 2000) } From eab0eb2a983aa3880d0e91eb3ee1fc72abb03e09 Mon Sep 17 00:00:00 2001 From: rumgrum Date: Tue, 11 Jun 2024 07:16:37 +0300 Subject: [PATCH 09/15] refactor(was-code service): catch db code fetch exceptions --- src/services/wasm-codes/wasm-code.service.ts | 29 +++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/services/wasm-codes/wasm-code.service.ts b/src/services/wasm-codes/wasm-code.service.ts index 0fad4565..b2e07431 100644 --- a/src/services/wasm-codes/wasm-code.service.ts +++ b/src/services/wasm-codes/wasm-code.service.ts @@ -88,20 +88,23 @@ export class WasmCodeService implements WasmCodeAdapter { } async reloadWasmCodeIdsFromDB(): Promise { - const wasmCodesFromDB = await WasmCodeKey.findAllWithIds() - - const wasmCodes = wasmCodesFromDB.map( - (wasmCodeKey: WasmCodeKey) => - new WasmCode( - wasmCodeKey.codeKey, - wasmCodeKey.codeKeyIds.map( - (wasmCodeKeyId: WasmCodeKeyId) => wasmCodeKeyId.codeKeyId + try { + const wasmCodesFromDB = await WasmCodeKey.findAllWithIds() + + const wasmCodes = wasmCodesFromDB.map( + (wasmCodeKey: WasmCodeKey) => + new WasmCode( + wasmCodeKey.codeKey, + wasmCodeKey.codeKeyIds.map( + (wasmCodeKeyId: WasmCodeKeyId) => wasmCodeKeyId.codeKeyId + ) ) - ) - ) - - this.resetWasmCodes() - this.addWasmCodes(wasmCodes) + ) + this.resetWasmCodes() + this.addWasmCodes(wasmCodes) + } catch (error) { + console.error('Failed to reload wasm code ids from DB:', error) + } } startUpdater(): void { From 2f7191937cf0764c1ac18513b46a7d38104a9c5f Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Wed, 12 Jun 2024 02:39:12 -0400 Subject: [PATCH 10/15] clean up, merge config code IDs with DB, and got tests to pass --- src/core/config.ts | 26 ---- src/core/env.ts | 5 +- src/core/types.ts | 2 +- src/data/meilisearch/daos.ts | 13 +- src/data/transformers/index.ts | 74 +++++----- ...40612062819-seed-wasm-codes-from-config.ts | 28 ++++ src/db/models/WasmCodeKey.ts | 14 +- src/db/models/WasmCodeKeyId.ts | 8 +- src/scripts/console.ts | 6 +- src/scripts/export/process.ts | 11 +- src/scripts/export/trace.ts | 4 + src/scripts/preCompute.ts | 5 +- src/scripts/revalidate.ts | 10 +- src/scripts/searchUpdate.ts | 4 + src/scripts/transform.ts | 9 +- src/server/routes/indexer/computer.ts | 7 +- src/server/serve.ts | 5 + src/services/wasm-codes/types.ts | 4 +- src/services/wasm-codes/wasm-code.adapter.ts | 3 +- .../wasm-codes/wasm-code.service.test.ts | 16 +-- src/services/wasm-codes/wasm-code.service.ts | 136 +++++++++++------- src/test/setup.ts | 4 + 22 files changed, 219 insertions(+), 175 deletions(-) create mode 100644 src/db/migrations/20240612062819-seed-wasm-codes-from-config.ts diff --git a/src/core/config.ts b/src/core/config.ts index 38ba7be0..7780a04d 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -1,8 +1,6 @@ import * as fs from 'fs' import path from 'path' -import { WasmCodeService } from '@/services/wasm-codes' - import { Config } from './types' // Constants. @@ -25,27 +23,3 @@ export const loadConfig = (configOverride?: string) => { return config } - -export const updateConfigWasmCodes = async (configToUpdate?: Config) => { - const wasmCodeService = await WasmCodeService.setUpInstance() - updateConfigCodeIds(wasmCodeService.exportWasmCodes()) - - if (configToUpdate) { - configToUpdate.codeIds = config.codeIds - } - - return configToUpdate -} - -export const updateConfigCodeIds = async ( - codeIds: Record -): Promise => { - config.codeIds = codeIds -} - -/** - * Get code IDs for a list of keys in the config. - */ -export const getCodeIdsForKeys = (...keys: string[]): number[] => { - return WasmCodeService.getInstance().findWasmCodeIdsByKeys(...keys) ?? [] -} diff --git a/src/core/env.ts b/src/core/env.ts index 08d08db7..3685236d 100644 --- a/src/core/env.ts +++ b/src/core/env.ts @@ -13,7 +13,6 @@ import { } from '@/db' import { WasmCodeService } from '@/services/wasm-codes' -import { getCodeIdsForKeys, loadConfig } from './config' import { Cache, DbType, @@ -898,13 +897,15 @@ export const getEnv = ({ return contract?.json } + const getCodeIdsForKeys = (...keys: string[]): number[] => + WasmCodeService.getInstance().findWasmCodeIdsByKeys(...keys) + const contractMatchesCodeIdKeys: FormulaContractMatchesCodeIdKeysGetter = async (contractAddress, ...keys) => { const codeId = (await getContract(contractAddress))?.codeId return codeId !== undefined && getCodeIdsForKeys(...keys).includes(codeId) } - const config = loadConfig() // Tries to find the code ID of this contract in the code ID keys and returns // the first match. const getCodeIdKeyForContract: FormulaCodeIdKeyForContractGetter = async ( diff --git a/src/core/types.ts b/src/core/types.ts index ee6c82c5..cf522f50 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -47,7 +47,7 @@ export type Config = { apiKey?: string } // Map some arbitary string to a list of code IDs. - codeIds?: Record + codeIds?: Partial> // If present, sets up Sentry error reporting. sentryDsn?: string diff --git a/src/data/meilisearch/daos.ts b/src/data/meilisearch/daos.ts index 52c5bb5e..aaf8656e 100644 --- a/src/data/meilisearch/daos.ts +++ b/src/data/meilisearch/daos.ts @@ -1,7 +1,6 @@ import { Op, Sequelize } from 'sequelize' import { getEnv } from '@/core' -import { getCodeIdsForKeys } from '@/core/config' import { ContractEnv, FormulaType, @@ -9,6 +8,7 @@ import { MeilisearchIndexer, } from '@/core/types' import { Contract, WasmStateEvent, WasmStateEventTransformation } from '@/db' +import { WasmCodeService } from '@/services/wasm-codes' import { getDaoAddressForProposalModule } from '../webhooks/utils' @@ -60,7 +60,8 @@ export const daos: MeilisearchIndexer = { ) }, getBulkUpdates: async () => { - const codeIds = getCodeIdsForKeys('dao-core') + const codeIds = + WasmCodeService.getInstance().findWasmCodeIdsByKeys('dao-core') if (!codeIds.length) { return [] } @@ -136,8 +137,12 @@ export const daoProposals: MeilisearchIndexer = { ) }, getBulkUpdates: async () => { - const singleCodeIds = getCodeIdsForKeys('dao-proposal-single') - const multipleCodeIds = getCodeIdsForKeys('dao-proposal-multiple') + const singleCodeIds = WasmCodeService.getInstance().findWasmCodeIdsByKeys( + 'dao-proposal-single' + ) + const multipleCodeIds = WasmCodeService.getInstance().findWasmCodeIdsByKeys( + 'dao-proposal-multiple' + ) if (singleCodeIds.length + multipleCodeIds.length === 0) { return [] } diff --git a/src/data/transformers/index.ts b/src/data/transformers/index.ts index a70df1b7..77c31737 100644 --- a/src/data/transformers/index.ts +++ b/src/data/transformers/index.ts @@ -43,51 +43,49 @@ export const getProcessedTransformers = ( ...transformerMakers.map((maker) => maker(config)), ] - processedTransformers = _transformers.map(({ filter, ...webhook }) => { - const allCodeIds = WasmCodeService.getInstance().findWasmCodeIdsByKeys( - ...(filter.codeIdsKeys ?? []) - ) + processedTransformers = _transformers.map(({ filter, ...webhook }) => ({ + ...webhook, + filter: (event) => { + let match = true - return { - ...webhook, - filter: (event) => { - let match = true + const allCodeIds = WasmCodeService.getInstance().findWasmCodeIdsByKeys( + ...(filter.codeIdsKeys ?? []) + ) - if (allCodeIds?.length) { - match &&= allCodeIds.includes(event.codeId) - } + if (allCodeIds.length) { + match &&= allCodeIds.includes(event.codeId) + } - if (match && filter.contractAddresses?.length) { - match &&= filter.contractAddresses.includes(event.contractAddress) - } + if (match && filter.contractAddresses?.length) { + match &&= filter.contractAddresses.includes(event.contractAddress) + } - if (match && filter.matches) { - // Wrap in try/catch in case a transformer errors. Don't want to - // prevent other events from transforming. - try { - match &&= filter.matches(event) - } catch (error) { - console.error( - `Error matching transformer for event ${event.blockHeight}/${event.contractAddress}/${event.key}: ${error}` - ) - Sentry.captureException(error, { - tags: { - type: 'failed-transformer-match', - }, - extra: { - event, - }, - }) + if (match && filter.matches) { + // Wrap in try/catch in case a transformer errors. Don't want to + // prevent other events from transforming. + try { + match &&= filter.matches(event) + } catch (error) { + console.error( + `Error matching transformer for event ${event.blockHeight}/${event.contractAddress}/${event.key}: ${error}` + ) + Sentry.captureException(error, { + tags: { + type: 'failed-transformer-match', + }, + extra: { + event, + }, + }) - // On error, do not match. - match = false - } + // On error, do not match. + match = false } + } - return match - }, - } - }) + return match + }, + })) } return processedTransformers diff --git a/src/db/migrations/20240612062819-seed-wasm-codes-from-config.ts b/src/db/migrations/20240612062819-seed-wasm-codes-from-config.ts new file mode 100644 index 00000000..024fc235 --- /dev/null +++ b/src/db/migrations/20240612062819-seed-wasm-codes-from-config.ts @@ -0,0 +1,28 @@ +import { QueryInterface } from 'sequelize' + +import { WasmCodeService } from '@/services/wasm-codes' + +module.exports = { + async up(queryInterface: QueryInterface) { + // loads from config automatically + const codes = (await WasmCodeService.setUpInstance()).getWasmCodes() + + await queryInterface.bulkInsert( + 'WasmCodeKeys', + codes.map(({ codeKey }) => ({ codeKey })) + ) + await queryInterface.bulkInsert( + 'WasmCodeKeyIds', + codes.flatMap(({ codeKey, codeIds }) => + codeIds.map((codeKeyId) => ({ + codeKey, + codeKeyId, + })) + ) + ) + }, + async down(queryInterface: QueryInterface) { + await queryInterface.bulkDelete('WasmCodeKeyIds', {}) + await queryInterface.bulkDelete('WasmCodeKeys', {}) + }, +} diff --git a/src/db/models/WasmCodeKey.ts b/src/db/models/WasmCodeKey.ts index cd91a1ae..f2cf86a1 100644 --- a/src/db/models/WasmCodeKey.ts +++ b/src/db/models/WasmCodeKey.ts @@ -50,18 +50,20 @@ export class WasmCodeKey extends Model { }) } - static async createWasmCode( + static async createFromKeyAndIds( codeKey: string, codeKeyId: number | number[] ): Promise { await WasmCodeKey.upsert({ codeKey }) - const arrayCodeKeyId = Array.isArray(codeKeyId) ? codeKeyId : [codeKeyId] - await Promise.all( - arrayCodeKeyId.map(async (codeId: number) => { - await WasmCodeKeyId.upsert({ codeKeyId: codeId, codeKey }) - }) + const arrayCodeKeyId = Array.isArray(codeKeyId) ? codeKeyId : [codeKeyId] + await WasmCodeKeyId.bulkCreate( + arrayCodeKeyId.map((codeKeyId) => ({ codeKeyId, codeKey })), + { + ignoreDuplicates: true, + } ) + return WasmCodeKey.findByKeyIncludeIds(codeKey) } } diff --git a/src/db/models/WasmCodeKeyId.ts b/src/db/models/WasmCodeKeyId.ts index adc8641e..aa54e095 100644 --- a/src/db/models/WasmCodeKeyId.ts +++ b/src/db/models/WasmCodeKeyId.ts @@ -27,11 +27,5 @@ export class WasmCodeKeyId extends Model { declare codeKeyId: number @BelongsTo(() => WasmCodeKey, 'codeKey') - declare codeKeyIds: WasmCodeKey[] - - static async findAllWithKeyCode(): Promise { - return WasmCodeKeyId.findAll({ - include: WasmCodeKey, - }) - } + declare codeKeys: WasmCodeKey[] } diff --git a/src/scripts/console.ts b/src/scripts/console.ts index c8960f70..f0de166e 100644 --- a/src/scripts/console.ts +++ b/src/scripts/console.ts @@ -4,6 +4,7 @@ import { Context } from 'vm' import { Command } from 'commander' import { Op, Sequelize, fn } from 'sequelize' +import * as core from '@/core' import { loadConfig } from '@/core/config' import { DbType } from '@/core/types' import { loadDb } from '@/db' @@ -46,7 +47,10 @@ const main = async () => { }) // ADD TO CONTEXT - setupImport(Models) + setupImport({ + ...core, + ...Models, + }) // START REPL const r = repl.start('> ') diff --git a/src/scripts/export/process.ts b/src/scripts/export/process.ts index 7a788283..897d0f86 100644 --- a/src/scripts/export/process.ts +++ b/src/scripts/export/process.ts @@ -1,13 +1,9 @@ import * as Sentry from '@sentry/node' import { Command } from 'commander' -import { - DbType, - getBullWorker, - loadConfig, - updateConfigWasmCodes, -} from '@/core' +import { DbType, getBullWorker, loadConfig } from '@/core' import { State, loadDb } from '@/db' +import { WasmCodeService } from '@/services/wasm-codes' import { workerMakers } from './workers' @@ -49,7 +45,8 @@ const main = async () => { type: DbType.Accounts, }) - await updateConfigWasmCodes(config) + // Set up wasm code service. + await WasmCodeService.setUpInstance() // Initialize state. await State.createSingletonIfMissing() diff --git a/src/scripts/export/trace.ts b/src/scripts/export/trace.ts index 0b74c778..b47d1a12 100644 --- a/src/scripts/export/trace.ts +++ b/src/scripts/export/trace.ts @@ -16,6 +16,7 @@ import { } from '@/core' import { State, loadDb } from '@/db' import { setupMeilisearch } from '@/ms' +import { WasmCodeService } from '@/services/wasm-codes' import { handlerMakers } from './handlers' import { ExportQueueData, TracedEvent, TracedEventWithBlockTime } from './types' @@ -85,6 +86,9 @@ const trace = async () => { type: DbType.Data, }) + // Set up wasm code service. + await WasmCodeService.setUpInstance() + // Initialize state. await State.createSingletonIfMissing() diff --git a/src/scripts/preCompute.ts b/src/scripts/preCompute.ts index 69733d05..f755f0ce 100644 --- a/src/scripts/preCompute.ts +++ b/src/scripts/preCompute.ts @@ -64,7 +64,7 @@ export const main = async () => { const options = program.opts() // Load config with config option. - const config = loadConfig(options.config) + loadConfig(options.config) let args: Record = {} if (options.args) { @@ -88,6 +88,9 @@ export const main = async () => { throw new Error('No state found.') } + // Set up wasm code service. + await WasmCodeService.setUpInstance() + let addresses: string[] if (options.targets) { diff --git a/src/scripts/revalidate.ts b/src/scripts/revalidate.ts index 836d3a2e..a439b7f3 100644 --- a/src/scripts/revalidate.ts +++ b/src/scripts/revalidate.ts @@ -50,22 +50,22 @@ const main = async () => { const start = Date.now() // Load config with config option. - const config = loadConfig(_config) + loadConfig(_config) // Load DB on start. const sequelize = await loadDb() + // Set up wasm code service. + await WasmCodeService.setUpInstance() + let latestId = initial - 1 let updated = 0 let replaced = 0 const formulasReplaced = new Set() - const codeIdsKeysFromStr: string[] = - WasmCodeService.getInstance().extractWasmCodeKeys(codeIdsKeys) ?? [] - const codeIds = WasmCodeService.getInstance().findWasmCodeIdsByKeys( - ...codeIdsKeysFromStr + ...WasmCodeService.extractWasmCodeKeys(codeIdsKeys) ) ?? [] const contracts = diff --git a/src/scripts/searchUpdate.ts b/src/scripts/searchUpdate.ts index bb318e64..f30ebb3a 100644 --- a/src/scripts/searchUpdate.ts +++ b/src/scripts/searchUpdate.ts @@ -3,6 +3,7 @@ import { Command } from 'commander' import { loadConfig } from '@/core/config' import { loadDb } from '@/db' import { setupMeilisearch, updateIndexes } from '@/ms' +import { WasmCodeService } from '@/services/wasm-codes' const main = async () => { // Parse arguments. @@ -24,6 +25,9 @@ const main = async () => { // Connect to db. const sequelize = await loadDb() + // Set up wasm code service. + await WasmCodeService.setUpInstance() + try { // Setup meilisearch. await setupMeilisearch() diff --git a/src/scripts/transform.ts b/src/scripts/transform.ts index 6602b098..45d8c489 100644 --- a/src/scripts/transform.ts +++ b/src/scripts/transform.ts @@ -59,11 +59,14 @@ const main = async () => { console.log(`\n[${new Date().toISOString()}] Transforming existing events...`) // Load config with config option. - const config = loadConfig(_config) + loadConfig(_config) // Load DB on start. const sequelize = await loadDb() + // Set up wasm code service. + await WasmCodeService.setUpInstance() + let processed = 0 let computationsUpdated = 0 let computationsDestroyed = 0 @@ -75,11 +78,9 @@ const main = async () => { } : {} - const codeIdsKeysFromStr = - WasmCodeService.getInstance().extractWasmCodeKeys(codeIdsKeys) ?? [] const codeIds = WasmCodeService.getInstance().findWasmCodeIdsByKeys( - ...codeIdsKeysFromStr + ...WasmCodeService.extractWasmCodeKeys(codeIdsKeys) ) ?? [] if (typeof codeIdsKeys === 'string' && codeIds.length === 0) { diff --git a/src/server/routes/indexer/computer.ts b/src/server/routes/indexer/computer.ts index 2c443cca..17154048 100644 --- a/src/server/routes/indexer/computer.ts +++ b/src/server/routes/indexer/computer.ts @@ -9,7 +9,6 @@ import { computeRange, getBlockForTime, getFirstBlock, - loadConfig, typeIsFormulaType, validateBlockString, } from '@/core' @@ -31,8 +30,6 @@ const testRateLimit = new Map() const testCooldownSeconds = 10 export const computer: Router.Middleware = async (ctx) => { - const config = loadConfig() - const { block: _block, blocks: _blocks, @@ -297,9 +294,7 @@ export const computer: Router.Middleware = async (ctx) => { if (typedFormula.formula.filter.codeIdsKeys?.length) { const codeIdKeys = typedFormula.formula.filter.codeIdsKeys const allCodeIds = - WasmCodeService.getInstance().findWasmCodeIdsByKeys( - ...codeIdKeys - ) ?? [] + WasmCodeService.getInstance().findWasmCodeIdsByKeys(...codeIdKeys) allowed &&= allCodeIds.includes(contract.codeId) } diff --git a/src/server/serve.ts b/src/server/serve.ts index 2998863f..b212b8ec 100644 --- a/src/server/serve.ts +++ b/src/server/serve.ts @@ -8,6 +8,7 @@ import Koa from 'koa' import { loadConfig } from '@/core/config' import { DbType } from '@/core/types' import { closeDb, loadDb } from '@/db' +import { WasmCodeService } from '@/services/wasm-codes' import { setupRouter } from './routes' import { captureSentryException } from './sentry' @@ -77,12 +78,16 @@ const main = async () => { await loadDb({ type: DbType.Accounts, }) + // Only connect to data if we're not serving the accounts API (i.e. we're // serving indexer data). if (!accounts) { await loadDb({ type: DbType.Data, }) + + // Set up wasm code service. + await WasmCodeService.setUpInstance() } if (!options.port || isNaN(options.port)) { diff --git a/src/services/wasm-codes/types.ts b/src/services/wasm-codes/types.ts index f0b36b37..7ebf0259 100644 --- a/src/services/wasm-codes/types.ts +++ b/src/services/wasm-codes/types.ts @@ -1,14 +1,14 @@ export class WasmCode { constructor( private readonly _codeKey: string, - private readonly _codeIds: number[] | undefined + private readonly _codeIds: number[] ) {} get codeKey(): string { return this._codeKey } - get codeIds(): number[] | undefined { + get codeIds(): number[] { return this._codeIds } } diff --git a/src/services/wasm-codes/wasm-code.adapter.ts b/src/services/wasm-codes/wasm-code.adapter.ts index 1cb5b01a..254a58a6 100644 --- a/src/services/wasm-codes/wasm-code.adapter.ts +++ b/src/services/wasm-codes/wasm-code.adapter.ts @@ -2,8 +2,7 @@ import { WasmCode } from './types' export interface WasmCodeAdapter { getWasmCodes(): WasmCode[] - resetWasmCodes(): void - exportWasmCodes(): Record + exportWasmCodes(): Partial> findWasmCodeIdsByKeys(...keys: string[]): number[] findWasmCodeKeysById(id: number): string[] reloadWasmCodeIdsFromDB(): Promise diff --git a/src/services/wasm-codes/wasm-code.service.test.ts b/src/services/wasm-codes/wasm-code.service.test.ts index 78ae84aa..231a70c7 100644 --- a/src/services/wasm-codes/wasm-code.service.test.ts +++ b/src/services/wasm-codes/wasm-code.service.test.ts @@ -18,9 +18,9 @@ describe('WasmCodeService tests', () => { codeKey3: [1, 3, 5], } - await WasmCodeKey.createWasmCode('codeKey1', [1, 2, 3]) - await WasmCodeKey.createWasmCode('codeKey2', [4, 5, 6]) - await WasmCodeKey.createWasmCode('codeKey3', [1, 3, 5]) + await WasmCodeKey.createFromKeyAndIds('codeKey1', [1, 2, 3]) + await WasmCodeKey.createFromKeyAndIds('codeKey2', [4, 5, 6]) + await WasmCodeKey.createFromKeyAndIds('codeKey3', [1, 3, 5]) wasmCodeService = await WasmCodeService.setUpInstance() @@ -60,9 +60,9 @@ describe('WasmCodeService tests', () => { cascade: true, }) - await WasmCodeKey.createWasmCode('codeKey1', 1) - await WasmCodeKey.createWasmCode('codeKey2', [2, 3]) - await WasmCodeKey.createWasmCode('codeKey3', []) + await WasmCodeKey.createFromKeyAndIds('codeKey1', 1) + await WasmCodeKey.createFromKeyAndIds('codeKey2', [2, 3]) + await WasmCodeKey.createFromKeyAndIds('codeKey3', []) await wasmCodeService.reloadWasmCodeIdsFromDB() @@ -74,11 +74,9 @@ describe('WasmCodeService tests', () => { expect(wasmCodeService.getWasmCodes()).toEqual(wasmCodes) + await WasmCodeKey.createFromKeyAndIds('codeKey4', []) await wasmCodeService.reloadWasmCodeIdsFromDB() - expect(wasmCodeService.getWasmCodes()).toEqual(wasmCodes) - await WasmCodeKey.createWasmCode('codeKey4', []) - await wasmCodeService.reloadWasmCodeIdsFromDB() expect(wasmCodeService.getWasmCodes()).toEqual([ ...wasmCodes, new WasmCode('codeKey4', []), diff --git a/src/services/wasm-codes/wasm-code.service.ts b/src/services/wasm-codes/wasm-code.service.ts index b2e07431..9e046f7d 100644 --- a/src/services/wasm-codes/wasm-code.service.ts +++ b/src/services/wasm-codes/wasm-code.service.ts @@ -1,16 +1,43 @@ -import { updateConfigCodeIds } from '@/core/config' +import { Config, loadConfig } from '@/core' import { WasmCodeKey } from '@/db/models/WasmCodeKey' import { WasmCodeKeyId } from '@/db/models/WasmCodeKeyId' import { WasmCode } from './types' import { WasmCodeAdapter } from './wasm-code.adapter' export class WasmCodeService implements WasmCodeAdapter { - private wasmCodes: WasmCode[] = [] - private interval: NodeJS.Timeout | undefined - + /** + * Singleton instance. + */ static instance: WasmCodeService - private constructor() {} + /** + * Wasm codes that are always added to the list, even when wasm codes are + * reloaded from the DB. + */ + private defaultWasmCodes: WasmCode[] + + /** + * List of all active wasm codes, including the defaults and those loaded from + * the DB. + */ + private wasmCodes: WasmCode[] + + /** + * Interval that updates the list of wasm codes from the DB. + */ + private refreshInterval: NodeJS.Timeout | undefined + + private constructor( + /** + * Wasm codes from the config. + */ + configWasmCodes: Config['codeIds'] + ) { + this.defaultWasmCodes = Object.entries(configWasmCodes || {}).flatMap( + ([key, codeIds]) => (codeIds ? new WasmCode(key, codeIds) : []) + ) + this.wasmCodes = [...this.defaultWasmCodes] + } static getInstance(): WasmCodeService { if (!this.instance) { @@ -25,39 +52,33 @@ export class WasmCodeService implements WasmCodeAdapter { if (this.instance) { return this.instance } - const wasmCodeService = new WasmCodeService() - await wasmCodeService.reloadWasmCodeIdsFromDB() - wasmCodeService.startUpdater() + + const config = loadConfig() + + const wasmCodeService = new WasmCodeService(config.codeIds) + await wasmCodeService.startUpdater() + return wasmCodeService } - resetWasmCodes(): void { - this.wasmCodes = [] - } + static extractWasmCodeKeys(input: any): string[] { + if (!input || typeof input !== 'string') { + return [] + } - addWasmCodes(wasmCodes: WasmCode[]): void { - this.wasmCodes.push(...wasmCodes) + return input.split(',').map((key: string) => key.trim()) } getWasmCodes(): WasmCode[] { return this.wasmCodes - .map((wasmCode: WasmCode) => { - wasmCode.codeIds?.sort() - return wasmCode - }) - .sort((a: WasmCode, b: WasmCode) => a.codeKey.localeCompare(b.codeKey)) } - exportWasmCodes(): Record { - return this.wasmCodes.reduce( - ( - acc: Record, - wasmCode: WasmCode - ): Record => ({ - ...acc, - [wasmCode.codeKey]: wasmCode.codeIds?.sort(), - }), - {} + exportWasmCodes(): Record { + return Object.fromEntries( + this.wasmCodes.map((wasmCode: WasmCode) => [ + wasmCode.codeKey, + wasmCode.codeIds, + ]) ) } @@ -71,27 +92,15 @@ export class WasmCodeService implements WasmCodeAdapter { findWasmCodeKeysById(codeId: number): string[] { return this.wasmCodes - .filter((wasmCode: WasmCode) => wasmCode.codeIds?.includes(codeId)) + .filter((wasmCode: WasmCode) => wasmCode.codeIds.includes(codeId)) .map((wasmCode: WasmCode) => wasmCode.codeKey) } - static extractWasmCodeKeys(input: any): string[] { - if (!input) { - return [] - } - - if (typeof input !== 'string') { - return [] - } - - return input.split(',').map((key: string) => key.trim()) - } - async reloadWasmCodeIdsFromDB(): Promise { try { const wasmCodesFromDB = await WasmCodeKey.findAllWithIds() - const wasmCodes = wasmCodesFromDB.map( + const dbWasmCodes = wasmCodesFromDB.map( (wasmCodeKey: WasmCodeKey) => new WasmCode( wasmCodeKey.codeKey, @@ -100,28 +109,47 @@ export class WasmCodeService implements WasmCodeAdapter { ) ) ) - this.resetWasmCodes() - this.addWasmCodes(wasmCodes) + + // Reset to defaults. + this.wasmCodes = [...this.defaultWasmCodes] + + // Merge from DB with existing keys. + for (const dbWasmCode of dbWasmCodes) { + let existing = this.wasmCodes.find( + (code) => code.codeKey === dbWasmCode.codeKey + ) + if (!existing) { + existing = new WasmCode(dbWasmCode.codeKey, []) + this.wasmCodes.push(existing) + } + + // Add non-existing code ids. + for (const codeId of dbWasmCode.codeIds) { + if (!existing.codeIds.includes(codeId)) { + existing.codeIds.push(codeId) + } + } + } } catch (error) { - console.error('Failed to reload wasm code ids from DB:', error) + console.error('Failed to reload wasm code IDs from DB:', error) } } - startUpdater(): void { - if (this.interval) { + async startUpdater(): Promise { + if (this.refreshInterval) { return } - this.interval = setInterval(async () => { - await this.reloadWasmCodeIdsFromDB() - await updateConfigCodeIds(this.exportWasmCodes()) - }, 2000) + this.refreshInterval = setInterval(this.reloadWasmCodeIdsFromDB, 60 * 1000) + + // Initial reload. + await this.reloadWasmCodeIdsFromDB() } stopUpdater(): void { - if (this.interval) { - clearInterval(this.interval) - this.interval = undefined + if (this.refreshInterval) { + clearInterval(this.refreshInterval) + this.refreshInterval = undefined } } } diff --git a/src/test/setup.ts b/src/test/setup.ts index 83b61365..c319cc2e 100644 --- a/src/test/setup.ts +++ b/src/test/setup.ts @@ -5,6 +5,7 @@ import { loadConfig } from '@/core/config' import { closeAllBullQueues } from '@/core/queues' import { DbType } from '@/core/types' import { closeDb, loadDb, setup } from '@/db' +import { WasmCodeService } from '@/services/wasm-codes' loadConfig() @@ -22,6 +23,9 @@ beforeEach(async () => { await setup(dataSequelize) await setup(accountsSequelize) + + // Set up wasm code service. + await WasmCodeService.setUpInstance() }) afterAll(async () => { From 294d30e78ff2d047b0235bfed115fb72de185907 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Wed, 12 Jun 2024 11:30:51 -0400 Subject: [PATCH 11/15] fixed tests --- .../wasm-codes/wasm-code.service.test.ts | 10 ++++++---- src/services/wasm-codes/wasm-code.service.ts | 18 ++++++++++++++---- src/test/setup.ts | 6 ++++-- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/services/wasm-codes/wasm-code.service.test.ts b/src/services/wasm-codes/wasm-code.service.test.ts index 231a70c7..8c245327 100644 --- a/src/services/wasm-codes/wasm-code.service.test.ts +++ b/src/services/wasm-codes/wasm-code.service.test.ts @@ -5,13 +5,17 @@ import { WasmCode } from './types' import { WasmCodeService } from './wasm-code.service' describe('WasmCodeService tests', () => { - let wasmCodeService: WasmCodeService + beforeAll(async () => { + await WasmCodeService.setUpInstance() + }) afterAll(() => { - wasmCodeService.stopUpdater() + WasmCodeService.getInstance().stopUpdater() }) test('WasmCodeService', async () => { + const wasmCodeService = WasmCodeService.getInstance() + const codeIds = { codeKey1: [1, 2, 3], codeKey2: [4, 5, 6], @@ -22,8 +26,6 @@ describe('WasmCodeService tests', () => { await WasmCodeKey.createFromKeyAndIds('codeKey2', [4, 5, 6]) await WasmCodeKey.createFromKeyAndIds('codeKey3', [1, 3, 5]) - wasmCodeService = await WasmCodeService.setUpInstance() - expect(wasmCodeService.getWasmCodes()).toEqual([ new WasmCode('codeKey1', [1, 2, 3]), new WasmCode('codeKey2', [4, 5, 6]), diff --git a/src/services/wasm-codes/wasm-code.service.ts b/src/services/wasm-codes/wasm-code.service.ts index 9e046f7d..ae945fc6 100644 --- a/src/services/wasm-codes/wasm-code.service.ts +++ b/src/services/wasm-codes/wasm-code.service.ts @@ -4,6 +4,7 @@ import { WasmCodeKeyId } from '@/db/models/WasmCodeKeyId' import { WasmCode } from './types' import { WasmCodeAdapter } from './wasm-code.adapter' + export class WasmCodeService implements WasmCodeAdapter { /** * Singleton instance. @@ -48,17 +49,26 @@ export class WasmCodeService implements WasmCodeAdapter { return this.instance } - static async setUpInstance(): Promise { + static async setUpInstance({ + withUpdater = true, + }: { + /** + * Whether or not to start the updater automatically. Defaults to true. + */ + withUpdater?: boolean + } = {}): Promise { if (this.instance) { return this.instance } const config = loadConfig() - const wasmCodeService = new WasmCodeService(config.codeIds) - await wasmCodeService.startUpdater() + this.instance = new WasmCodeService(config.codeIds) + if (withUpdater) { + await this.instance.startUpdater() + } - return wasmCodeService + return this.instance } static extractWasmCodeKeys(input: any): string[] { diff --git a/src/test/setup.ts b/src/test/setup.ts index c319cc2e..34b595ea 100644 --- a/src/test/setup.ts +++ b/src/test/setup.ts @@ -24,8 +24,10 @@ beforeEach(async () => { await setup(dataSequelize) await setup(accountsSequelize) - // Set up wasm code service. - await WasmCodeService.setUpInstance() + // Set up wasm code service without updater. + await WasmCodeService.setUpInstance({ + withUpdater: false, + }) }) afterAll(async () => { From 31ff1df9b734ac7d31f911d8f07e2eadf913ab53 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Wed, 12 Jun 2024 11:47:51 -0400 Subject: [PATCH 12/15] fixed tests again --- .../test/indexer/computer/formulas/index.ts | 9 +- src/services/wasm-codes/wasm-code.adapter.ts | 1 + .../wasm-codes/wasm-code.service.test.ts | 10 +- src/services/wasm-codes/wasm-code.service.ts | 132 +++++++++++++----- 4 files changed, 107 insertions(+), 45 deletions(-) diff --git a/src/server/test/indexer/computer/formulas/index.ts b/src/server/test/indexer/computer/formulas/index.ts index b7092410..f1b0e650 100644 --- a/src/server/test/indexer/computer/formulas/index.ts +++ b/src/server/test/indexer/computer/formulas/index.ts @@ -1,7 +1,7 @@ import request from 'supertest' -import { loadConfig } from '@/core' import { Contract } from '@/db' +import { WasmCode, WasmCodeService } from '@/services/wasm-codes' import { app } from '../../app' import { ComputerTestOptions } from '../types' @@ -22,9 +22,10 @@ export const loadFormulasTests = (options: ComputerTestOptions) => { loadWasmTests(options) it('filters contract by code IDs specified in formula', async () => { - loadConfig().codeIds = { - 'dao-core': [1, 2], - } + WasmCodeService.getInstance().addDefaultWasmCodes( + new WasmCode('dao-core', [1, 2]) + ) + options.mockFormula({ filter: { codeIdsKeys: ['not-dao-core'], diff --git a/src/services/wasm-codes/wasm-code.adapter.ts b/src/services/wasm-codes/wasm-code.adapter.ts index 254a58a6..768f7886 100644 --- a/src/services/wasm-codes/wasm-code.adapter.ts +++ b/src/services/wasm-codes/wasm-code.adapter.ts @@ -1,6 +1,7 @@ import { WasmCode } from './types' export interface WasmCodeAdapter { + addDefaultWasmCodes(...wasmCodes: WasmCode[]): void getWasmCodes(): WasmCode[] exportWasmCodes(): Partial> findWasmCodeIdsByKeys(...keys: string[]): number[] diff --git a/src/services/wasm-codes/wasm-code.service.test.ts b/src/services/wasm-codes/wasm-code.service.test.ts index 8c245327..b55f2199 100644 --- a/src/services/wasm-codes/wasm-code.service.test.ts +++ b/src/services/wasm-codes/wasm-code.service.test.ts @@ -6,11 +6,9 @@ import { WasmCodeService } from './wasm-code.service' describe('WasmCodeService tests', () => { beforeAll(async () => { - await WasmCodeService.setUpInstance() - }) - - afterAll(() => { - WasmCodeService.getInstance().stopUpdater() + await WasmCodeService.setUpInstance({ + withUpdater: false, + }) }) test('WasmCodeService', async () => { @@ -26,6 +24,8 @@ describe('WasmCodeService tests', () => { await WasmCodeKey.createFromKeyAndIds('codeKey2', [4, 5, 6]) await WasmCodeKey.createFromKeyAndIds('codeKey3', [1, 3, 5]) + await wasmCodeService.reloadWasmCodeIdsFromDB() + expect(wasmCodeService.getWasmCodes()).toEqual([ new WasmCode('codeKey1', [1, 2, 3]), new WasmCode('codeKey2', [4, 5, 6]), diff --git a/src/services/wasm-codes/wasm-code.service.ts b/src/services/wasm-codes/wasm-code.service.ts index ae945fc6..512042ff 100644 --- a/src/services/wasm-codes/wasm-code.service.ts +++ b/src/services/wasm-codes/wasm-code.service.ts @@ -40,6 +40,10 @@ export class WasmCodeService implements WasmCodeAdapter { this.wasmCodes = [...this.defaultWasmCodes] } + /** + * Return the singleton created by the setUpInstance method, throwing an error + * if not yet setup. + */ static getInstance(): WasmCodeService { if (!this.instance) { throw new Error( @@ -49,6 +53,12 @@ export class WasmCodeService implements WasmCodeAdapter { return this.instance } + /** + * Set up the WasmCodeService by loading defaults from the config and + * optionally starting the DB updater. + * + * Creates a singleton that is returned if already setup. + */ static async setUpInstance({ withUpdater = true, }: { @@ -71,6 +81,11 @@ export class WasmCodeService implements WasmCodeAdapter { return this.instance } + /** + * Parse wasm code keys from an arbitrary input. + * + * Used in CLI. + */ static extractWasmCodeKeys(input: any): string[] { if (!input || typeof input !== 'string') { return [] @@ -79,10 +94,51 @@ export class WasmCodeService implements WasmCodeAdapter { return input.split(',').map((key: string) => key.trim()) } + /** + * Merge two lists of wasm codes. + */ + private mergeWasmCodes(src: WasmCode[], dst: WasmCode[]): void { + // Merge from src into dst. + for (const { codeKey, codeIds } of src) { + let existing = dst.find((code) => code.codeKey === codeKey) + if (!existing) { + existing = new WasmCode(codeKey, []) + dst.push(existing) + } + + // Add non-existent code ids. + for (const codeId of codeIds) { + if (!existing.codeIds.includes(codeId)) { + existing.codeIds.push(codeId) + } + } + } + } + + /** + * Manually add new wasm codes, storing them in the default list so they stick + * around during DB updates. + * + * Used in tests. + */ + addDefaultWasmCodes(...wasmCodes: WasmCode[]): void { + // First store new wasm codes in default list so they stick around when the + // DB updates. + this.mergeWasmCodes(wasmCodes, this.defaultWasmCodes) + // Then merge into existing list. + this.mergeWasmCodes(wasmCodes, this.wasmCodes) + } + + /** + * Return a copy of the list of wasm codes. + */ getWasmCodes(): WasmCode[] { - return this.wasmCodes + return [...this.wasmCodes] } + /** + * Return a map of code key to code IDs. + */ exportWasmCodes(): Record { return Object.fromEntries( this.wasmCodes.map((wasmCode: WasmCode) => [ @@ -92,6 +148,9 @@ export class WasmCodeService implements WasmCodeAdapter { ) } + /** + * Find all code IDs for the list of keys. + */ findWasmCodeIdsByKeys(...keys: string[]): number[] { return keys.flatMap( (key: string) => @@ -100,62 +159,63 @@ export class WasmCodeService implements WasmCodeAdapter { ) } + /** + * Find all keys that contain the given code ID. + */ findWasmCodeKeysById(codeId: number): string[] { return this.wasmCodes .filter((wasmCode: WasmCode) => wasmCode.codeIds.includes(codeId)) .map((wasmCode: WasmCode) => wasmCode.codeKey) } + /** + * Reload wasm codes from DB, preserving the default list and removing any + * previously loaded from the DB that no longer exist. + */ async reloadWasmCodeIdsFromDB(): Promise { - try { - const wasmCodesFromDB = await WasmCodeKey.findAllWithIds() - - const dbWasmCodes = wasmCodesFromDB.map( - (wasmCodeKey: WasmCodeKey) => - new WasmCode( - wasmCodeKey.codeKey, - wasmCodeKey.codeKeyIds.map( - (wasmCodeKeyId: WasmCodeKeyId) => wasmCodeKeyId.codeKeyId - ) + const wasmCodesFromDB = await WasmCodeKey.findAllWithIds() + + const dbWasmCodes = wasmCodesFromDB.map( + (wasmCodeKey: WasmCodeKey) => + new WasmCode( + wasmCodeKey.codeKey, + wasmCodeKey.codeKeyIds.map( + (wasmCodeKeyId: WasmCodeKeyId) => wasmCodeKeyId.codeKeyId ) - ) - - // Reset to defaults. - this.wasmCodes = [...this.defaultWasmCodes] - - // Merge from DB with existing keys. - for (const dbWasmCode of dbWasmCodes) { - let existing = this.wasmCodes.find( - (code) => code.codeKey === dbWasmCode.codeKey ) - if (!existing) { - existing = new WasmCode(dbWasmCode.codeKey, []) - this.wasmCodes.push(existing) - } + ) - // Add non-existing code ids. - for (const codeId of dbWasmCode.codeIds) { - if (!existing.codeIds.includes(codeId)) { - existing.codeIds.push(codeId) - } - } - } - } catch (error) { - console.error('Failed to reload wasm code IDs from DB:', error) - } + // Reset to defaults. + this.wasmCodes = [...this.defaultWasmCodes] + + // Merge DB codes into list with defaults. + this.mergeWasmCodes(dbWasmCodes, this.wasmCodes) } + /** + * Start updating wasm codes from DB on a timer, if not already started. + */ async startUpdater(): Promise { if (this.refreshInterval) { return } - this.refreshInterval = setInterval(this.reloadWasmCodeIdsFromDB, 60 * 1000) - // Initial reload. await this.reloadWasmCodeIdsFromDB() + + // Start updater. + this.refreshInterval = setInterval(async () => { + try { + await this.reloadWasmCodeIdsFromDB() + } catch (error) { + console.error('Failed to reload wasm code IDs from DB:', error) + } + }, 60 * 1000) } + /** + * Stop updating wasm codes from DB on a timer, if started. + */ stopUpdater(): void { if (this.refreshInterval) { clearInterval(this.refreshInterval) From 69671dbaf1ff15944f4bff59b0cceffafcec3480 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Wed, 12 Jun 2024 17:09:20 -0400 Subject: [PATCH 13/15] make sure to stop wasm code service updater --- src/scripts/export/process.ts | 10 +++++++++- src/scripts/export/trace.ts | 7 ++++++- src/server/serve.ts | 13 +++++++++++-- src/services/wasm-codes/wasm-code.service.test.ts | 4 +--- src/services/wasm-codes/wasm-code.service.ts | 12 +++++++++--- src/test/setup.ts | 6 ++---- 6 files changed, 38 insertions(+), 14 deletions(-) diff --git a/src/scripts/export/process.ts b/src/scripts/export/process.ts index 897d0f86..ecb017a8 100644 --- a/src/scripts/export/process.ts +++ b/src/scripts/export/process.ts @@ -46,7 +46,9 @@ const main = async () => { }) // Set up wasm code service. - await WasmCodeService.setUpInstance() + await WasmCodeService.setUpInstance({ + withUpdater: true, + }) // Initialize state. await State.createSingletonIfMissing() @@ -93,8 +95,14 @@ const main = async () => { console.log('Shutting down after current worker jobs complete...') // Exit once all workers close. Promise.all(workers.map((worker) => worker.close())).then(async () => { + // Stop services. + WasmCodeService.getInstance().stopUpdater() + + // Close DB connections. await dataSequelize.close() await accountsSequelize.close() + + // Exit. process.exit(0) }) } diff --git a/src/scripts/export/trace.ts b/src/scripts/export/trace.ts index b47d1a12..20188100 100644 --- a/src/scripts/export/trace.ts +++ b/src/scripts/export/trace.ts @@ -87,7 +87,9 @@ const trace = async () => { }) // Set up wasm code service. - await WasmCodeService.setUpInstance() + await WasmCodeService.setUpInstance({ + withUpdater: true, + }) // Initialize state. await State.createSingletonIfMissing() @@ -662,6 +664,9 @@ const trace = async () => { await traceExporter + // Stop services. + WasmCodeService.getInstance().stopUpdater() + // Close database connection. await dataSequelize.close() diff --git a/src/server/serve.ts b/src/server/serve.ts index b212b8ec..6636d38a 100644 --- a/src/server/serve.ts +++ b/src/server/serve.ts @@ -72,6 +72,8 @@ setupRouter(app, { accounts, }) +let wasmCodeService: WasmCodeService | null = null + // Start. const main = async () => { // All servers need to connect to the accounts DB. @@ -87,7 +89,9 @@ const main = async () => { }) // Set up wasm code service. - await WasmCodeService.setUpInstance() + wasmCodeService = await WasmCodeService.setUpInstance({ + withUpdater: true, + }) } if (!options.port || isNaN(options.port)) { @@ -106,9 +110,14 @@ const main = async () => { main() -// On exit, close DB connection. +// On exit, stop services and close DB connection. const cleanup = async () => { console.log('Shutting down...') + + if (wasmCodeService) { + wasmCodeService.stopUpdater() + } + await closeDb() } diff --git a/src/services/wasm-codes/wasm-code.service.test.ts b/src/services/wasm-codes/wasm-code.service.test.ts index b55f2199..4e3feab0 100644 --- a/src/services/wasm-codes/wasm-code.service.test.ts +++ b/src/services/wasm-codes/wasm-code.service.test.ts @@ -6,9 +6,7 @@ import { WasmCodeService } from './wasm-code.service' describe('WasmCodeService tests', () => { beforeAll(async () => { - await WasmCodeService.setUpInstance({ - withUpdater: false, - }) + await WasmCodeService.setUpInstance() }) test('WasmCodeService', async () => { diff --git a/src/services/wasm-codes/wasm-code.service.ts b/src/services/wasm-codes/wasm-code.service.ts index 512042ff..b885e813 100644 --- a/src/services/wasm-codes/wasm-code.service.ts +++ b/src/services/wasm-codes/wasm-code.service.ts @@ -5,6 +5,12 @@ import { WasmCodeKeyId } from '@/db/models/WasmCodeKeyId' import { WasmCode } from './types' import { WasmCodeAdapter } from './wasm-code.adapter' +/** + * A service to manage wasm codes that are loaded from the DB. This is used by + * various systems throughout the indexer, such as transformers that filter and + * transform state events from specific contracts and webhooks that listen for + * on-chain events from specific contracts. + */ export class WasmCodeService implements WasmCodeAdapter { /** * Singleton instance. @@ -54,16 +60,16 @@ export class WasmCodeService implements WasmCodeAdapter { } /** - * Set up the WasmCodeService by loading defaults from the config and + * Set up the wasm code service by loading defaults from the config and * optionally starting the DB updater. * * Creates a singleton that is returned if already setup. */ static async setUpInstance({ - withUpdater = true, + withUpdater = false, }: { /** - * Whether or not to start the updater automatically. Defaults to true. + * Whether or not to start the updater automatically. Defaults to false. */ withUpdater?: boolean } = {}): Promise { diff --git a/src/test/setup.ts b/src/test/setup.ts index 34b595ea..c319cc2e 100644 --- a/src/test/setup.ts +++ b/src/test/setup.ts @@ -24,10 +24,8 @@ beforeEach(async () => { await setup(dataSequelize) await setup(accountsSequelize) - // Set up wasm code service without updater. - await WasmCodeService.setUpInstance({ - withUpdater: false, - }) + // Set up wasm code service. + await WasmCodeService.setUpInstance() }) afterAll(async () => { From 249d69013ba34d7fef4de0a5feb5ab015898ed8f Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Wed, 12 Jun 2024 17:15:18 -0400 Subject: [PATCH 14/15] add primary key to WasmCodeKeyIds table --- src/db/migrations/20240521022046-create-wasm-code-key-id.ts | 5 +++++ src/db/models/WasmCodeKeyId.ts | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/db/migrations/20240521022046-create-wasm-code-key-id.ts b/src/db/migrations/20240521022046-create-wasm-code-key-id.ts index 62415d78..275c2fb1 100644 --- a/src/db/migrations/20240521022046-create-wasm-code-key-id.ts +++ b/src/db/migrations/20240521022046-create-wasm-code-key-id.ts @@ -4,6 +4,11 @@ import { DataType } from 'sequelize-typescript' module.exports = { async up(queryInterface: QueryInterface) { await queryInterface.createTable('WasmCodeKeyIds', { + id: { + primaryKey: true, + autoIncrement: true, + type: DataType.INTEGER, + }, codeKey: { allowNull: false, type: DataType.STRING, diff --git a/src/db/models/WasmCodeKeyId.ts b/src/db/models/WasmCodeKeyId.ts index aa54e095..1db88e15 100644 --- a/src/db/models/WasmCodeKeyId.ts +++ b/src/db/models/WasmCodeKeyId.ts @@ -3,6 +3,7 @@ import { BelongsTo, Column, Model, + PrimaryKey, Table, } from 'sequelize-typescript' @@ -18,6 +19,10 @@ import { WasmCodeKey } from './WasmCodeKey' ], }) export class WasmCodeKeyId extends Model { + @PrimaryKey + @Column + declare id: number + @AllowNull(false) @Column declare codeKey: string From 461d4247854d036cd3fefc26ac8336184572601f Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Wed, 12 Jun 2024 17:23:46 -0400 Subject: [PATCH 15/15] added services to console --- src/scripts/console.ts | 2 ++ src/services/index.ts | 1 + 2 files changed, 3 insertions(+) create mode 100644 src/services/index.ts diff --git a/src/scripts/console.ts b/src/scripts/console.ts index f0de166e..225e14cc 100644 --- a/src/scripts/console.ts +++ b/src/scripts/console.ts @@ -9,6 +9,7 @@ import { loadConfig } from '@/core/config' import { DbType } from '@/core/types' import { loadDb } from '@/db' import * as Models from '@/db/models' +import * as Services from '@/services' // Global context available to repl. const context: Context = { @@ -50,6 +51,7 @@ const main = async () => { setupImport({ ...core, ...Models, + ...Services, }) // START REPL diff --git a/src/services/index.ts b/src/services/index.ts new file mode 100644 index 00000000..058e0acd --- /dev/null +++ b/src/services/index.ts @@ -0,0 +1 @@ +export { WasmCodeService } from './wasm-codes'