diff --git a/src/core/config.ts b/src/core/config.ts index e2e5ceb5..7780a04d 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -23,11 +23,3 @@ export const loadConfig = (configOverride?: string) => { return config } - -/** - * Get code IDs for a list of keys in the config. - */ -export const getCodeIdsForKeys = (...keys: string[]) => { - const config = loadConfig() - return keys.flatMap((key) => config.codeIds?.[key] ?? []) -} diff --git a/src/core/env.ts b/src/core/env.ts index 780bbf6a..3685236d 100644 --- a/src/core/env.ts +++ b/src/core/env.ts @@ -11,8 +11,8 @@ import { WasmTxEvent, loadDb, } from '@/db' +import { WasmCodeService } from '@/services/wasm-codes' -import { getCodeIdsForKeys, loadConfig } from './config' import { Cache, DbType, @@ -897,13 +897,15 @@ export const getEnv = ({ return contract?.json } + const getCodeIdsForKeys = (...keys: string[]): number[] => + WasmCodeService.getInstance().findWasmCodeIdsByKeys(...keys) + const contractMatchesCodeIdKeys: FormulaContractMatchesCodeIdKeysGetter = async (contractAddress, ...keys) => { const codeId = (await getContract(contractAddress))?.codeId return codeId !== undefined && getCodeIdsForKeys(...keys).includes(codeId) } - const config = loadConfig() // Tries to find the code ID of this contract in the code ID keys and returns // the first match. const getCodeIdKeyForContract: FormulaCodeIdKeyForContractGetter = async ( @@ -914,10 +916,7 @@ export const getEnv = ({ return } - const codeIdKeys = Object.entries(config.codeIds ?? {}).flatMap( - ([key, value]) => (value?.includes(codeId) ? [key] : []) - ) - return codeIdKeys[0] + return WasmCodeService.getInstance().findWasmCodeKeysById(codeId)[0] } const getSlashEvents: FormulaSlashEventsGetter = async ( diff --git a/src/core/types.ts b/src/core/types.ts index 03c65185..cf522f50 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -47,7 +47,8 @@ export type Config = { apiKey?: string } // Map some arbitary string to a list of code IDs. - codeIds?: Record + codeIds?: Partial> + // If present, sets up Sentry error reporting. sentryDsn?: string // Payment info. diff --git a/src/data/meilisearch/daos.ts b/src/data/meilisearch/daos.ts index 52c5bb5e..aaf8656e 100644 --- a/src/data/meilisearch/daos.ts +++ b/src/data/meilisearch/daos.ts @@ -1,7 +1,6 @@ import { Op, Sequelize } from 'sequelize' import { getEnv } from '@/core' -import { getCodeIdsForKeys } from '@/core/config' import { ContractEnv, FormulaType, @@ -9,6 +8,7 @@ import { MeilisearchIndexer, } from '@/core/types' import { Contract, WasmStateEvent, WasmStateEventTransformation } from '@/db' +import { WasmCodeService } from '@/services/wasm-codes' import { getDaoAddressForProposalModule } from '../webhooks/utils' @@ -60,7 +60,8 @@ export const daos: MeilisearchIndexer = { ) }, getBulkUpdates: async () => { - const codeIds = getCodeIdsForKeys('dao-core') + const codeIds = + WasmCodeService.getInstance().findWasmCodeIdsByKeys('dao-core') if (!codeIds.length) { return [] } @@ -136,8 +137,12 @@ export const daoProposals: MeilisearchIndexer = { ) }, getBulkUpdates: async () => { - const singleCodeIds = getCodeIdsForKeys('dao-proposal-single') - const multipleCodeIds = getCodeIdsForKeys('dao-proposal-multiple') + const singleCodeIds = WasmCodeService.getInstance().findWasmCodeIdsByKeys( + 'dao-proposal-single' + ) + const multipleCodeIds = WasmCodeService.getInstance().findWasmCodeIdsByKeys( + 'dao-proposal-multiple' + ) if (singleCodeIds.length + multipleCodeIds.length === 0) { return [] } diff --git a/src/data/transformers/index.ts b/src/data/transformers/index.ts index 94503237..77c31737 100644 --- a/src/data/transformers/index.ts +++ b/src/data/transformers/index.ts @@ -6,6 +6,7 @@ import { Transformer, TransformerMaker, } from '@/core' +import { WasmCodeService } from '@/services/wasm-codes' import common from './common' import dao from './dao' @@ -42,51 +43,49 @@ export const getProcessedTransformers = ( ...transformerMakers.map((maker) => maker(config)), ] - processedTransformers = _transformers.map(({ filter, ...webhook }) => { - const allCodeIds = filter.codeIdsKeys?.flatMap( - (key) => config.codeIds?.[key] ?? [] - ) + processedTransformers = _transformers.map(({ filter, ...webhook }) => ({ + ...webhook, + filter: (event) => { + let match = true - return { - ...webhook, - filter: (event) => { - let match = true + const allCodeIds = WasmCodeService.getInstance().findWasmCodeIdsByKeys( + ...(filter.codeIdsKeys ?? []) + ) - if (allCodeIds?.length) { - match &&= allCodeIds.includes(event.codeId) - } + if (allCodeIds.length) { + match &&= allCodeIds.includes(event.codeId) + } - if (match && filter.contractAddresses?.length) { - match &&= filter.contractAddresses.includes(event.contractAddress) - } + if (match && filter.contractAddresses?.length) { + match &&= filter.contractAddresses.includes(event.contractAddress) + } - if (match && filter.matches) { - // Wrap in try/catch in case a transformer errors. Don't want to - // prevent other events from transforming. - try { - match &&= filter.matches(event) - } catch (error) { - console.error( - `Error matching transformer for event ${event.blockHeight}/${event.contractAddress}/${event.key}: ${error}` - ) - Sentry.captureException(error, { - tags: { - type: 'failed-transformer-match', - }, - extra: { - event, - }, - }) + if (match && filter.matches) { + // Wrap in try/catch in case a transformer errors. Don't want to + // prevent other events from transforming. + try { + match &&= filter.matches(event) + } catch (error) { + console.error( + `Error matching transformer for event ${event.blockHeight}/${event.contractAddress}/${event.key}: ${error}` + ) + Sentry.captureException(error, { + tags: { + type: 'failed-transformer-match', + }, + extra: { + event, + }, + }) - // On error, do not match. - match = false - } + // On error, do not match. + match = false } + } - return match - }, - } - }) + return match + }, + })) } return processedTransformers diff --git a/src/data/webhooks/index.ts b/src/data/webhooks/index.ts index aa46f80e..bd1681fb 100644 --- a/src/data/webhooks/index.ts +++ b/src/data/webhooks/index.ts @@ -2,6 +2,7 @@ import * as Sentry from '@sentry/node' import { Config, ProcessedWebhook, Webhook, WebhookMaker } from '@/core' import { State, WasmStateEvent } from '@/db' +import { WasmCodeService } from '@/services/wasm-codes' import { makeProposalCreated } from './discordNotifier' import { makeIndexerCwReceiptPaid } from './indexerCwReceipt' @@ -59,8 +60,8 @@ export const getProcessedWebhooks = ( .filter((webhook): webhook is Webhook => !!webhook) processedWebhooks = _webhooks.map(({ filter, ...webhook }) => { - const allCodeIds = filter.codeIdsKeys?.flatMap( - (key) => config.codeIds?.[key] ?? [] + const allCodeIds = WasmCodeService.getInstance().findWasmCodeIdsByKeys( + ...(filter.codeIdsKeys ?? []) ) return { diff --git a/src/db/connection.ts b/src/db/connection.ts index 68a29c7e..98772f5b 100644 --- a/src/db/connection.ts +++ b/src/db/connection.ts @@ -21,6 +21,8 @@ import { StakingSlashEvent, State, Validator, + WasmCodeKey, + WasmCodeKeyId, WasmStateEvent, WasmStateEventTransformation, WasmTxEvent, @@ -47,6 +49,8 @@ const getModelsForType = (type: DbType): SequelizeOptions['models'] => StakingSlashEvent, State, Validator, + WasmCodeKey, + WasmCodeKeyId, WasmStateEvent, WasmStateEventTransformation, WasmTxEvent, diff --git a/src/db/migrations/20240521012355-create-wasm-code-key.ts b/src/db/migrations/20240521012355-create-wasm-code-key.ts new file mode 100644 index 00000000..c8e5ea56 --- /dev/null +++ b/src/db/migrations/20240521012355-create-wasm-code-key.ts @@ -0,0 +1,35 @@ +import { QueryInterface, fn } from 'sequelize' +import { DataType } from 'sequelize-typescript' + +module.exports = { + async up(queryInterface: QueryInterface) { + await queryInterface.createTable('WasmCodeKeys', { + codeKey: { + primaryKey: true, + allowNull: false, + type: DataType.STRING, + }, + description: { + allowNull: true, + type: DataType.TEXT, + }, + createdAt: { + allowNull: false, + type: DataType.DATE, + defaultValue: fn('NOW'), + }, + updatedAt: { + allowNull: false, + type: DataType.DATE, + defaultValue: fn('NOW'), + }, + }) + await queryInterface.addIndex('WasmCodeKeys', { + unique: true, + fields: ['codeKey'], + }) + }, + async down(queryInterface: QueryInterface) { + await queryInterface.dropTable('WasmCodeKeys') + }, +} diff --git a/src/db/migrations/20240521022046-create-wasm-code-key-id.ts b/src/db/migrations/20240521022046-create-wasm-code-key-id.ts new file mode 100644 index 00000000..275c2fb1 --- /dev/null +++ b/src/db/migrations/20240521022046-create-wasm-code-key-id.ts @@ -0,0 +1,47 @@ +import { QueryInterface, fn } from 'sequelize' +import { DataType } from 'sequelize-typescript' + +module.exports = { + async up(queryInterface: QueryInterface) { + await queryInterface.createTable('WasmCodeKeyIds', { + id: { + primaryKey: true, + autoIncrement: true, + type: DataType.INTEGER, + }, + codeKey: { + allowNull: false, + type: DataType.STRING, + references: { + model: 'WasmCodeKeys', + key: 'codeKey', + }, + }, + codeKeyId: { + allowNull: false, + type: DataType.INTEGER, + }, + description: { + allowNull: true, + type: DataType.TEXT, + }, + createdAt: { + allowNull: false, + type: DataType.DATE, + defaultValue: fn('NOW'), + }, + updatedAt: { + allowNull: false, + type: DataType.DATE, + defaultValue: fn('NOW'), + }, + }) + await queryInterface.addIndex('WasmCodeKeyIds', { + unique: true, + fields: ['codeKey', 'codeKeyId'], + }) + }, + async down(queryInterface: QueryInterface) { + await queryInterface.dropTable('WasmCodeKeyIds') + }, +} diff --git a/src/db/migrations/20240612062819-seed-wasm-codes-from-config.ts b/src/db/migrations/20240612062819-seed-wasm-codes-from-config.ts new file mode 100644 index 00000000..024fc235 --- /dev/null +++ b/src/db/migrations/20240612062819-seed-wasm-codes-from-config.ts @@ -0,0 +1,28 @@ +import { QueryInterface } from 'sequelize' + +import { WasmCodeService } from '@/services/wasm-codes' + +module.exports = { + async up(queryInterface: QueryInterface) { + // loads from config automatically + const codes = (await WasmCodeService.setUpInstance()).getWasmCodes() + + await queryInterface.bulkInsert( + 'WasmCodeKeys', + codes.map(({ codeKey }) => ({ codeKey })) + ) + await queryInterface.bulkInsert( + 'WasmCodeKeyIds', + codes.flatMap(({ codeKey, codeIds }) => + codeIds.map((codeKeyId) => ({ + codeKey, + codeKeyId, + })) + ) + ) + }, + async down(queryInterface: QueryInterface) { + await queryInterface.bulkDelete('WasmCodeKeyIds', {}) + await queryInterface.bulkDelete('WasmCodeKeys', {}) + }, +} diff --git a/src/db/models/Contract.ts b/src/db/models/Contract.ts index 98024b97..a41e1009 100644 --- a/src/db/models/Contract.ts +++ b/src/db/models/Contract.ts @@ -7,8 +7,8 @@ import { Table, } from 'sequelize-typescript' -import { loadConfig } from '@/core/config' import { ContractJson } from '@/core/types' +import { WasmCodeService } from '@/services/wasm-codes' @Table({ timestamps: true, @@ -53,8 +53,8 @@ export class Contract extends Model { * from the config. */ matchesCodeIdKeys(...keys: string[]): boolean { - const config = loadConfig() - const codeIds = keys.flatMap((key) => config.codeIds?.[key] ?? []) + const codeIds = + WasmCodeService.getInstance().findWasmCodeIdsByKeys(...keys) ?? [] return codeIds.includes(this.codeId) } } diff --git a/src/db/models/WasmCodeKey.ts b/src/db/models/WasmCodeKey.ts new file mode 100644 index 00000000..f2cf86a1 --- /dev/null +++ b/src/db/models/WasmCodeKey.ts @@ -0,0 +1,69 @@ +import { + AllowNull, + Column, + DataType, + HasMany, + Model, + PrimaryKey, + Table, +} from 'sequelize-typescript' + +import { WasmCodeKeyId } from './WasmCodeKeyId' + +@Table({ + timestamps: true, + indexes: [ + { + unique: true, + fields: ['codeKey'], + }, + ], +}) +export class WasmCodeKey extends Model { + @PrimaryKey + @Column + declare codeKey: string + + @AllowNull(true) + @Column(DataType.TEXT) + declare description: string + + @HasMany(() => WasmCodeKeyId, 'codeKey') + declare codeKeyIds: WasmCodeKeyId[] + + async associateCodeKeyIds(codeKeyIds: WasmCodeKeyId[]): Promise { + await this.$add('codeKeyIds', codeKeyIds) + } + + static async findByKeyIncludeIds( + codeKey: string + ): Promise { + return WasmCodeKey.findOne({ + where: { codeKey }, + include: WasmCodeKeyId, + }) + } + + static async findAllWithIds(): Promise { + return WasmCodeKey.findAll({ + include: WasmCodeKeyId, + }) + } + + static async createFromKeyAndIds( + codeKey: string, + codeKeyId: number | number[] + ): Promise { + await WasmCodeKey.upsert({ codeKey }) + + const arrayCodeKeyId = Array.isArray(codeKeyId) ? codeKeyId : [codeKeyId] + await WasmCodeKeyId.bulkCreate( + arrayCodeKeyId.map((codeKeyId) => ({ codeKeyId, codeKey })), + { + ignoreDuplicates: true, + } + ) + + return WasmCodeKey.findByKeyIncludeIds(codeKey) + } +} diff --git a/src/db/models/WasmCodeKeyId.ts b/src/db/models/WasmCodeKeyId.ts new file mode 100644 index 00000000..1db88e15 --- /dev/null +++ b/src/db/models/WasmCodeKeyId.ts @@ -0,0 +1,36 @@ +import { + AllowNull, + BelongsTo, + Column, + Model, + PrimaryKey, + Table, +} from 'sequelize-typescript' + +import { WasmCodeKey } from './WasmCodeKey' + +@Table({ + timestamps: true, + indexes: [ + { + unique: true, + fields: ['codeKey', 'codeKeyId'], + }, + ], +}) +export class WasmCodeKeyId extends Model { + @PrimaryKey + @Column + declare id: number + + @AllowNull(false) + @Column + declare codeKey: string + + @AllowNull(false) + @Column + declare codeKeyId: number + + @BelongsTo(() => WasmCodeKey, 'codeKey') + declare codeKeys: WasmCodeKey[] +} diff --git a/src/db/models/index.ts b/src/db/models/index.ts index ace0bbe8..22097b8f 100644 --- a/src/db/models/index.ts +++ b/src/db/models/index.ts @@ -3,9 +3,9 @@ export * from './AccountCodeIdSet' export * from './AccountKey' export * from './AccountKeyCredit' export * from './AccountWebhook' +export * from './AccountWebhookCodeIdSet' export * from './AccountWebhookEvent' export * from './AccountWebhookEventAttempt' -export * from './AccountWebhookCodeIdSet' export * from './BankStateEvent' export * from './Computation' export * from './ComputationDependency' @@ -15,6 +15,8 @@ export * from './GovStateEvent' export * from './StakingSlashEvent' export * from './State' export * from './Validator' +export * from './WasmCodeKey' +export * from './WasmCodeKeyId' export * from './WasmStateEvent' export * from './WasmStateEventTransformation' export * from './WasmTxEvent' diff --git a/src/scripts/console.ts b/src/scripts/console.ts index c8960f70..225e14cc 100644 --- a/src/scripts/console.ts +++ b/src/scripts/console.ts @@ -4,10 +4,12 @@ import { Context } from 'vm' import { Command } from 'commander' import { Op, Sequelize, fn } from 'sequelize' +import * as core from '@/core' import { loadConfig } from '@/core/config' import { DbType } from '@/core/types' import { loadDb } from '@/db' import * as Models from '@/db/models' +import * as Services from '@/services' // Global context available to repl. const context: Context = { @@ -46,7 +48,11 @@ const main = async () => { }) // ADD TO CONTEXT - setupImport(Models) + setupImport({ + ...core, + ...Models, + ...Services, + }) // START REPL const r = repl.start('> ') diff --git a/src/scripts/export/process.ts b/src/scripts/export/process.ts index 3a0f8bbd..ecb017a8 100644 --- a/src/scripts/export/process.ts +++ b/src/scripts/export/process.ts @@ -3,6 +3,7 @@ import { Command } from 'commander' import { DbType, getBullWorker, loadConfig } from '@/core' import { State, loadDb } from '@/db' +import { WasmCodeService } from '@/services/wasm-codes' import { workerMakers } from './workers' @@ -44,6 +45,11 @@ const main = async () => { type: DbType.Accounts, }) + // Set up wasm code service. + await WasmCodeService.setUpInstance({ + withUpdater: true, + }) + // Initialize state. await State.createSingletonIfMissing() @@ -89,13 +95,18 @@ const main = async () => { console.log('Shutting down after current worker jobs complete...') // Exit once all workers close. Promise.all(workers.map((worker) => worker.close())).then(async () => { + // Stop services. + WasmCodeService.getInstance().stopUpdater() + + // Close DB connections. await dataSequelize.close() await accountsSequelize.close() + + // Exit. process.exit(0) }) } }) - // Tell pm2 we're ready. if (process.send) { process.send('ready') diff --git a/src/scripts/export/trace.ts b/src/scripts/export/trace.ts index 0b74c778..20188100 100644 --- a/src/scripts/export/trace.ts +++ b/src/scripts/export/trace.ts @@ -16,6 +16,7 @@ import { } from '@/core' import { State, loadDb } from '@/db' import { setupMeilisearch } from '@/ms' +import { WasmCodeService } from '@/services/wasm-codes' import { handlerMakers } from './handlers' import { ExportQueueData, TracedEvent, TracedEventWithBlockTime } from './types' @@ -85,6 +86,11 @@ const trace = async () => { type: DbType.Data, }) + // Set up wasm code service. + await WasmCodeService.setUpInstance({ + withUpdater: true, + }) + // Initialize state. await State.createSingletonIfMissing() @@ -658,6 +664,9 @@ const trace = async () => { await traceExporter + // Stop services. + WasmCodeService.getInstance().stopUpdater() + // Close database connection. await dataSequelize.close() diff --git a/src/scripts/preCompute.ts b/src/scripts/preCompute.ts index 2c7cc307..f755f0ce 100644 --- a/src/scripts/preCompute.ts +++ b/src/scripts/preCompute.ts @@ -12,6 +12,7 @@ import { } from '@/core' import { getTypedFormula } from '@/data' import { Computation, Contract, State, loadDb } from '@/db' +import { WasmCodeService } from '@/services/wasm-codes' export const main = async () => { // Parse arguments. @@ -63,7 +64,7 @@ export const main = async () => { const options = program.opts() // Load config with config option. - const config = loadConfig(options.config) + loadConfig(options.config) let args: Record = {} if (options.args) { @@ -87,6 +88,9 @@ export const main = async () => { throw new Error('No state found.') } + // Set up wasm code service. + await WasmCodeService.setUpInstance() + let addresses: string[] if (options.targets) { @@ -94,9 +98,9 @@ export const main = async () => { } else if (options.ids?.length || options.codeIdsKeys?.length) { const codeIds = [ ...(options.ids || []), - ...(options.codeIdsKeys || []).flatMap( - (key: string) => config.codeIds?.[key] - ), + ...(WasmCodeService.getInstance().findWasmCodeIdsByKeys( + options.codeIdsKeys || [] + ) ?? []), ] addresses = ( await Contract.findAll({ diff --git a/src/scripts/revalidate.ts b/src/scripts/revalidate.ts index 4c81b1bc..a439b7f3 100644 --- a/src/scripts/revalidate.ts +++ b/src/scripts/revalidate.ts @@ -6,6 +6,7 @@ import { Op } from 'sequelize' import { loadConfig } from '@/core' import { Computation, Contract, loadDb } from '@/db' +import { WasmCodeService } from '@/services/wasm-codes' const LOADER_MAP = ['—', '\\', '|', '/'] @@ -49,19 +50,24 @@ const main = async () => { const start = Date.now() // Load config with config option. - const config = loadConfig(_config) + loadConfig(_config) // Load DB on start. const sequelize = await loadDb() + // Set up wasm code service. + await WasmCodeService.setUpInstance() + let latestId = initial - 1 let updated = 0 let replaced = 0 const formulasReplaced = new Set() - const codeIds = ( - codeIdsKeys && typeof codeIdsKeys === 'string' ? codeIdsKeys.split(',') : [] - ).flatMap((key) => config.codeIds?.[key] ?? []) + const codeIds = + WasmCodeService.getInstance().findWasmCodeIdsByKeys( + ...WasmCodeService.extractWasmCodeKeys(codeIdsKeys) + ) ?? [] + const contracts = codeIds.length > 0 ? await Contract.findAll({ diff --git a/src/scripts/searchUpdate.ts b/src/scripts/searchUpdate.ts index bb318e64..f30ebb3a 100644 --- a/src/scripts/searchUpdate.ts +++ b/src/scripts/searchUpdate.ts @@ -3,6 +3,7 @@ import { Command } from 'commander' import { loadConfig } from '@/core/config' import { loadDb } from '@/db' import { setupMeilisearch, updateIndexes } from '@/ms' +import { WasmCodeService } from '@/services/wasm-codes' const main = async () => { // Parse arguments. @@ -24,6 +25,9 @@ const main = async () => { // Connect to db. const sequelize = await loadDb() + // Set up wasm code service. + await WasmCodeService.setUpInstance() + try { // Setup meilisearch. await setupMeilisearch() diff --git a/src/scripts/transform.ts b/src/scripts/transform.ts index 46dd3890..45d8c489 100644 --- a/src/scripts/transform.ts +++ b/src/scripts/transform.ts @@ -9,6 +9,7 @@ import { loadDb, updateComputationValidityDependentOnChanges, } from '@/db' +import { WasmCodeService } from '@/services/wasm-codes' const LOADER_MAP = ['—', '\\', '|', '/'] @@ -58,11 +59,14 @@ const main = async () => { console.log(`\n[${new Date().toISOString()}] Transforming existing events...`) // Load config with config option. - const config = loadConfig(_config) + loadConfig(_config) // Load DB on start. const sequelize = await loadDb() + // Set up wasm code service. + await WasmCodeService.setUpInstance() + let processed = 0 let computationsUpdated = 0 let computationsDestroyed = 0 @@ -74,9 +78,11 @@ const main = async () => { } : {} - const codeIds = ( - codeIdsKeys && typeof codeIdsKeys === 'string' ? codeIdsKeys.split(',') : [] - ).flatMap((key) => config.codeIds?.[key] ?? []) + const codeIds = + WasmCodeService.getInstance().findWasmCodeIdsByKeys( + ...WasmCodeService.extractWasmCodeKeys(codeIdsKeys) + ) ?? [] + if (typeof codeIdsKeys === 'string' && codeIds.length === 0) { throw new Error('No code IDs found in config') } diff --git a/src/server/routes/indexer/computer.ts b/src/server/routes/indexer/computer.ts index 5dee3278..17154048 100644 --- a/src/server/routes/indexer/computer.ts +++ b/src/server/routes/indexer/computer.ts @@ -9,7 +9,6 @@ import { computeRange, getBlockForTime, getFirstBlock, - loadConfig, typeIsFormulaType, validateBlockString, } from '@/core' @@ -22,6 +21,7 @@ import { State, Validator, } from '@/db' +import { WasmCodeService } from '@/services/wasm-codes' import { captureSentryException } from '../../sentry' @@ -30,8 +30,6 @@ const testRateLimit = new Map() const testCooldownSeconds = 10 export const computer: Router.Middleware = async (ctx) => { - const config = loadConfig() - const { block: _block, blocks: _blocks, @@ -294,9 +292,9 @@ export const computer: Router.Middleware = async (ctx) => { let allowed = true if (typedFormula.formula.filter.codeIdsKeys?.length) { - const allCodeIds = typedFormula.formula.filter.codeIdsKeys.flatMap( - (key) => config.codeIds?.[key] ?? [] - ) + const codeIdKeys = typedFormula.formula.filter.codeIdsKeys + const allCodeIds = + WasmCodeService.getInstance().findWasmCodeIdsByKeys(...codeIdKeys) allowed &&= allCodeIds.includes(contract.codeId) } diff --git a/src/server/serve.ts b/src/server/serve.ts index 2998863f..6636d38a 100644 --- a/src/server/serve.ts +++ b/src/server/serve.ts @@ -8,6 +8,7 @@ import Koa from 'koa' import { loadConfig } from '@/core/config' import { DbType } from '@/core/types' import { closeDb, loadDb } from '@/db' +import { WasmCodeService } from '@/services/wasm-codes' import { setupRouter } from './routes' import { captureSentryException } from './sentry' @@ -71,18 +72,26 @@ setupRouter(app, { accounts, }) +let wasmCodeService: WasmCodeService | null = null + // Start. const main = async () => { // All servers need to connect to the accounts DB. await loadDb({ type: DbType.Accounts, }) + // Only connect to data if we're not serving the accounts API (i.e. we're // serving indexer data). if (!accounts) { await loadDb({ type: DbType.Data, }) + + // Set up wasm code service. + wasmCodeService = await WasmCodeService.setUpInstance({ + withUpdater: true, + }) } if (!options.port || isNaN(options.port)) { @@ -101,9 +110,14 @@ const main = async () => { main() -// On exit, close DB connection. +// On exit, stop services and close DB connection. const cleanup = async () => { console.log('Shutting down...') + + if (wasmCodeService) { + wasmCodeService.stopUpdater() + } + await closeDb() } diff --git a/src/server/test/indexer/computer/formulas/index.ts b/src/server/test/indexer/computer/formulas/index.ts index b7092410..f1b0e650 100644 --- a/src/server/test/indexer/computer/formulas/index.ts +++ b/src/server/test/indexer/computer/formulas/index.ts @@ -1,7 +1,7 @@ import request from 'supertest' -import { loadConfig } from '@/core' import { Contract } from '@/db' +import { WasmCode, WasmCodeService } from '@/services/wasm-codes' import { app } from '../../app' import { ComputerTestOptions } from '../types' @@ -22,9 +22,10 @@ export const loadFormulasTests = (options: ComputerTestOptions) => { loadWasmTests(options) it('filters contract by code IDs specified in formula', async () => { - loadConfig().codeIds = { - 'dao-core': [1, 2], - } + WasmCodeService.getInstance().addDefaultWasmCodes( + new WasmCode('dao-core', [1, 2]) + ) + options.mockFormula({ filter: { codeIdsKeys: ['not-dao-core'], diff --git a/src/services/index.ts b/src/services/index.ts new file mode 100644 index 00000000..058e0acd --- /dev/null +++ b/src/services/index.ts @@ -0,0 +1 @@ +export { WasmCodeService } from './wasm-codes' diff --git a/src/services/wasm-codes/index.ts b/src/services/wasm-codes/index.ts new file mode 100644 index 00000000..4de6ccbb --- /dev/null +++ b/src/services/wasm-codes/index.ts @@ -0,0 +1,3 @@ +export * from './wasm-code.adapter' +export * from './wasm-code.service' +export * from './types' diff --git a/src/services/wasm-codes/types.ts b/src/services/wasm-codes/types.ts new file mode 100644 index 00000000..7ebf0259 --- /dev/null +++ b/src/services/wasm-codes/types.ts @@ -0,0 +1,14 @@ +export class WasmCode { + constructor( + private readonly _codeKey: string, + private readonly _codeIds: number[] + ) {} + + get codeKey(): string { + return this._codeKey + } + + get codeIds(): number[] { + return this._codeIds + } +} diff --git a/src/services/wasm-codes/wasm-code.adapter.ts b/src/services/wasm-codes/wasm-code.adapter.ts new file mode 100644 index 00000000..768f7886 --- /dev/null +++ b/src/services/wasm-codes/wasm-code.adapter.ts @@ -0,0 +1,10 @@ +import { WasmCode } from './types' + +export interface WasmCodeAdapter { + addDefaultWasmCodes(...wasmCodes: WasmCode[]): void + getWasmCodes(): WasmCode[] + exportWasmCodes(): Partial> + findWasmCodeIdsByKeys(...keys: string[]): number[] + findWasmCodeKeysById(id: number): string[] + reloadWasmCodeIdsFromDB(): Promise +} diff --git a/src/services/wasm-codes/wasm-code.service.test.ts b/src/services/wasm-codes/wasm-code.service.test.ts new file mode 100644 index 00000000..4e3feab0 --- /dev/null +++ b/src/services/wasm-codes/wasm-code.service.test.ts @@ -0,0 +1,85 @@ +import { WasmCodeKeyId } from '@/db' +import { WasmCodeKey } from '@/db/models/WasmCodeKey' + +import { WasmCode } from './types' +import { WasmCodeService } from './wasm-code.service' + +describe('WasmCodeService tests', () => { + beforeAll(async () => { + await WasmCodeService.setUpInstance() + }) + + test('WasmCodeService', async () => { + const wasmCodeService = WasmCodeService.getInstance() + + const codeIds = { + codeKey1: [1, 2, 3], + codeKey2: [4, 5, 6], + codeKey3: [1, 3, 5], + } + + await WasmCodeKey.createFromKeyAndIds('codeKey1', [1, 2, 3]) + await WasmCodeKey.createFromKeyAndIds('codeKey2', [4, 5, 6]) + await WasmCodeKey.createFromKeyAndIds('codeKey3', [1, 3, 5]) + + await wasmCodeService.reloadWasmCodeIdsFromDB() + + expect(wasmCodeService.getWasmCodes()).toEqual([ + new WasmCode('codeKey1', [1, 2, 3]), + new WasmCode('codeKey2', [4, 5, 6]), + new WasmCode('codeKey3', [1, 3, 5]), + ]) + + expect(wasmCodeService.exportWasmCodes()).toEqual(codeIds) + + expect(wasmCodeService.findWasmCodeIdsByKeys('codeKey1')).toEqual([1, 2, 3]) + expect(wasmCodeService.findWasmCodeIdsByKeys('codeKey2')).toEqual([4, 5, 6]) + expect( + wasmCodeService.findWasmCodeIdsByKeys('codeKey1', 'codeKey2') + ).toEqual([1, 2, 3, 4, 5, 6]) + + expect(wasmCodeService.findWasmCodeKeysById(1)).toEqual([ + 'codeKey1', + 'codeKey3', + ]) + expect(wasmCodeService.findWasmCodeKeysById(4)).toEqual(['codeKey2']) + expect(wasmCodeService.findWasmCodeKeysById(7)).toEqual([]) + + expect(WasmCodeService.extractWasmCodeKeys(undefined)).toEqual([]) + expect(WasmCodeService.extractWasmCodeKeys('')).toEqual([]) + expect(WasmCodeService.extractWasmCodeKeys('codeKey1')).toEqual([ + 'codeKey1', + ]) + expect(WasmCodeService.extractWasmCodeKeys('codeKey1,codeKey2')).toEqual([ + 'codeKey1', + 'codeKey2', + ]) + + await WasmCodeKeyId.truncate() + await WasmCodeKey.truncate({ + cascade: true, + }) + + await WasmCodeKey.createFromKeyAndIds('codeKey1', 1) + await WasmCodeKey.createFromKeyAndIds('codeKey2', [2, 3]) + await WasmCodeKey.createFromKeyAndIds('codeKey3', []) + + await wasmCodeService.reloadWasmCodeIdsFromDB() + + const wasmCodes = [ + new WasmCode('codeKey1', [1]), + new WasmCode('codeKey2', [2, 3]), + new WasmCode('codeKey3', []), + ] + + expect(wasmCodeService.getWasmCodes()).toEqual(wasmCodes) + + await WasmCodeKey.createFromKeyAndIds('codeKey4', []) + await wasmCodeService.reloadWasmCodeIdsFromDB() + + expect(wasmCodeService.getWasmCodes()).toEqual([ + ...wasmCodes, + new WasmCode('codeKey4', []), + ]) + }) +}) diff --git a/src/services/wasm-codes/wasm-code.service.ts b/src/services/wasm-codes/wasm-code.service.ts new file mode 100644 index 00000000..b885e813 --- /dev/null +++ b/src/services/wasm-codes/wasm-code.service.ts @@ -0,0 +1,231 @@ +import { Config, loadConfig } from '@/core' +import { WasmCodeKey } from '@/db/models/WasmCodeKey' +import { WasmCodeKeyId } from '@/db/models/WasmCodeKeyId' + +import { WasmCode } from './types' +import { WasmCodeAdapter } from './wasm-code.adapter' + +/** + * A service to manage wasm codes that are loaded from the DB. This is used by + * various systems throughout the indexer, such as transformers that filter and + * transform state events from specific contracts and webhooks that listen for + * on-chain events from specific contracts. + */ +export class WasmCodeService implements WasmCodeAdapter { + /** + * Singleton instance. + */ + static instance: WasmCodeService + + /** + * Wasm codes that are always added to the list, even when wasm codes are + * reloaded from the DB. + */ + private defaultWasmCodes: WasmCode[] + + /** + * List of all active wasm codes, including the defaults and those loaded from + * the DB. + */ + private wasmCodes: WasmCode[] + + /** + * Interval that updates the list of wasm codes from the DB. + */ + private refreshInterval: NodeJS.Timeout | undefined + + private constructor( + /** + * Wasm codes from the config. + */ + configWasmCodes: Config['codeIds'] + ) { + this.defaultWasmCodes = Object.entries(configWasmCodes || {}).flatMap( + ([key, codeIds]) => (codeIds ? new WasmCode(key, codeIds) : []) + ) + this.wasmCodes = [...this.defaultWasmCodes] + } + + /** + * Return the singleton created by the setUpInstance method, throwing an error + * if not yet setup. + */ + static getInstance(): WasmCodeService { + if (!this.instance) { + throw new Error( + 'WasmCodeService not initialized because WasmCodeService.setUpInstance was never called' + ) + } + return this.instance + } + + /** + * Set up the wasm code service by loading defaults from the config and + * optionally starting the DB updater. + * + * Creates a singleton that is returned if already setup. + */ + static async setUpInstance({ + withUpdater = false, + }: { + /** + * Whether or not to start the updater automatically. Defaults to false. + */ + withUpdater?: boolean + } = {}): Promise { + if (this.instance) { + return this.instance + } + + const config = loadConfig() + + this.instance = new WasmCodeService(config.codeIds) + if (withUpdater) { + await this.instance.startUpdater() + } + + return this.instance + } + + /** + * Parse wasm code keys from an arbitrary input. + * + * Used in CLI. + */ + static extractWasmCodeKeys(input: any): string[] { + if (!input || typeof input !== 'string') { + return [] + } + + return input.split(',').map((key: string) => key.trim()) + } + + /** + * Merge two lists of wasm codes. + */ + private mergeWasmCodes(src: WasmCode[], dst: WasmCode[]): void { + // Merge from src into dst. + for (const { codeKey, codeIds } of src) { + let existing = dst.find((code) => code.codeKey === codeKey) + if (!existing) { + existing = new WasmCode(codeKey, []) + dst.push(existing) + } + + // Add non-existent code ids. + for (const codeId of codeIds) { + if (!existing.codeIds.includes(codeId)) { + existing.codeIds.push(codeId) + } + } + } + } + + /** + * Manually add new wasm codes, storing them in the default list so they stick + * around during DB updates. + * + * Used in tests. + */ + addDefaultWasmCodes(...wasmCodes: WasmCode[]): void { + // First store new wasm codes in default list so they stick around when the + // DB updates. + this.mergeWasmCodes(wasmCodes, this.defaultWasmCodes) + // Then merge into existing list. + this.mergeWasmCodes(wasmCodes, this.wasmCodes) + } + + /** + * Return a copy of the list of wasm codes. + */ + getWasmCodes(): WasmCode[] { + return [...this.wasmCodes] + } + + /** + * Return a map of code key to code IDs. + */ + exportWasmCodes(): Record { + return Object.fromEntries( + this.wasmCodes.map((wasmCode: WasmCode) => [ + wasmCode.codeKey, + wasmCode.codeIds, + ]) + ) + } + + /** + * Find all code IDs for the list of keys. + */ + findWasmCodeIdsByKeys(...keys: string[]): number[] { + return keys.flatMap( + (key: string) => + this.wasmCodes.find((wasmCode: WasmCode) => wasmCode.codeKey === key) + ?.codeIds ?? [] + ) + } + + /** + * Find all keys that contain the given code ID. + */ + findWasmCodeKeysById(codeId: number): string[] { + return this.wasmCodes + .filter((wasmCode: WasmCode) => wasmCode.codeIds.includes(codeId)) + .map((wasmCode: WasmCode) => wasmCode.codeKey) + } + + /** + * Reload wasm codes from DB, preserving the default list and removing any + * previously loaded from the DB that no longer exist. + */ + async reloadWasmCodeIdsFromDB(): Promise { + const wasmCodesFromDB = await WasmCodeKey.findAllWithIds() + + const dbWasmCodes = wasmCodesFromDB.map( + (wasmCodeKey: WasmCodeKey) => + new WasmCode( + wasmCodeKey.codeKey, + wasmCodeKey.codeKeyIds.map( + (wasmCodeKeyId: WasmCodeKeyId) => wasmCodeKeyId.codeKeyId + ) + ) + ) + + // Reset to defaults. + this.wasmCodes = [...this.defaultWasmCodes] + + // Merge DB codes into list with defaults. + this.mergeWasmCodes(dbWasmCodes, this.wasmCodes) + } + + /** + * Start updating wasm codes from DB on a timer, if not already started. + */ + async startUpdater(): Promise { + if (this.refreshInterval) { + return + } + + // Initial reload. + await this.reloadWasmCodeIdsFromDB() + + // Start updater. + this.refreshInterval = setInterval(async () => { + try { + await this.reloadWasmCodeIdsFromDB() + } catch (error) { + console.error('Failed to reload wasm code IDs from DB:', error) + } + }, 60 * 1000) + } + + /** + * Stop updating wasm codes from DB on a timer, if started. + */ + stopUpdater(): void { + if (this.refreshInterval) { + clearInterval(this.refreshInterval) + this.refreshInterval = undefined + } + } +} diff --git a/src/test/setup.ts b/src/test/setup.ts index 83b61365..c319cc2e 100644 --- a/src/test/setup.ts +++ b/src/test/setup.ts @@ -5,6 +5,7 @@ import { loadConfig } from '@/core/config' import { closeAllBullQueues } from '@/core/queues' import { DbType } from '@/core/types' import { closeDb, loadDb, setup } from '@/db' +import { WasmCodeService } from '@/services/wasm-codes' loadConfig() @@ -22,6 +23,9 @@ beforeEach(async () => { await setup(dataSequelize) await setup(accountsSequelize) + + // Set up wasm code service. + await WasmCodeService.setUpInstance() }) afterAll(async () => {