diff --git a/packages/common-near/CHANGELOG.md b/packages/common-near/CHANGELOG.md index 7cc7bf57..88d3a4a6 100644 --- a/packages/common-near/CHANGELOG.md +++ b/packages/common-near/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Update dependencies and apply changes to match (#109) ## [3.4.0] - 2024-04-10 ### Changed diff --git a/packages/common-near/package.json b/packages/common-near/package.json index 3baa0b30..6fa93e4e 100644 --- a/packages/common-near/package.json +++ b/packages/common-near/package.json @@ -14,7 +14,7 @@ "main": "dist/index.js", "license": "GPL-3.0", "dependencies": { - "@subql/common": "^3.5.0", + "@subql/common": "^3.5.1", "@subql/types-near": "workspace:*", "js-yaml": "^4.1.0", "reflect-metadata": "^0.1.13" diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md index ebe86805..1ef45206 100644 --- a/packages/node/CHANGELOG.md +++ b/packages/node/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Update dependencies and apply changes to match (#109) ## [3.10.0] - 2024-04-10 ### Changed diff --git a/packages/node/package.json b/packages/node/package.json index 9793e4a0..492db6c7 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -25,9 +25,9 @@ "@nestjs/event-emitter": "^2.0.0", "@nestjs/platform-express": "^9.4.0", "@nestjs/schedule": "^3.0.1", - "@subql/common": "^3.5.0", + "@subql/common": "^3.5.1", "@subql/common-near": "workspace:*", - "@subql/node-core": "^8.0.1", + "@subql/node-core": "^10.0.0", "@subql/types-near": "workspace:*", "cron-converter": "^1.0.2", "eventemitter2": "^6.4.5", diff --git a/packages/node/src/indexer/api.service.ts b/packages/node/src/indexer/api.service.ts index 3407726e..bc8a68c1 100644 --- a/packages/node/src/indexer/api.service.ts +++ b/packages/node/src/indexer/api.service.ts @@ -69,13 +69,8 @@ export class ApiService extends BaseApiService< ); } - await this.createConnections( - network, - (endpoint) => NearApiConnection.create(endpoint, this.fetchBlocksBatches), - async (connection: NearApiConnection) => { - const api = connection.unsafeApi; - return (await api.status()).chain_id; - }, + await this.createConnections(network, (endpoint) => + NearApiConnection.create(endpoint, this.fetchBlocksBatches), ); return this; diff --git a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts index 399f0d53..6fe161cd 100644 --- a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts @@ -5,7 +5,6 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { NodeConfig, - SmartBatchService, StoreCacheService, StoreService, IProjectService, @@ -20,7 +19,6 @@ import { SubqueryProject, } from '../../configure/SubqueryProject'; import { ApiService } from '../api.service'; -import { DynamicDsService } from '../dynamic-ds.service'; import { IndexerManager } from '../indexer.manager'; import { BlockContent } from '../types'; @@ -40,24 +38,20 @@ export class BlockDispatcherService @Inject('IProjectService') projectService: IProjectService, @Inject('IProjectUpgradeService') projectUpgradeService: IProjectUpgradeService, - smartBatchService: SmartBatchService, storeService: StoreService, storeCacheService: StoreCacheService, poiSyncService: PoiSyncService, @Inject('ISubqueryProject') project: SubqueryProject, - dynamicDsService: DynamicDsService, ) { super( nodeConfig, eventEmitter, projectService, projectUpgradeService, - smartBatchService, storeService, storeCacheService, poiSyncService, project, - dynamicDsService, async (blockNums: number[]): Promise[]> => { //filter out null values, they represent blocks that were not available in chain return (await this.apiService.fetchBlocks(blockNums)).filter( diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts index 0583833c..6daa57dc 100644 --- a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts @@ -6,7 +6,6 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { NodeConfig, - SmartBatchService, StoreService, StoreCacheService, IProjectService, @@ -43,7 +42,6 @@ export class WorkerBlockDispatcherService @Inject('IProjectService') projectService: IProjectService, @Inject('IProjectUpgradeService') projectUpgradeService: IProjectUpgradeService, - smartBatchService: SmartBatchService, cacheService: InMemoryCacheService, storeService: StoreService, storeCacheService: StoreCacheService, @@ -58,12 +56,10 @@ export class WorkerBlockDispatcherService eventEmitter, projectService, projectUpgradeService, - smartBatchService, storeService, storeCacheService, poiSyncService, project, - dynamicDsService, () => createIndexerWorker< IIndexerWorker, diff --git a/packages/node/src/indexer/ds-processor.service.ts b/packages/node/src/indexer/ds-processor.service.ts index ad792b98..bcb6dc88 100644 --- a/packages/node/src/indexer/ds-processor.service.ts +++ b/packages/node/src/indexer/ds-processor.service.ts @@ -9,71 +9,7 @@ import { NearDatasourceProcessor, } from '@subql/common-near'; import { BaseDsProcessorService } from '@subql/node-core'; -import { - SecondLayerHandlerProcessor_0_0_0, - SecondLayerHandlerProcessor_1_0_0, - NearCustomDatasource, - NearHandlerKind, - NearMapping, - NearCustomHandler, -} from '@subql/types-near'; - -export function isSecondLayerHandlerProcessor_0_0_0< - K extends NearHandlerKind, - F extends Record, - E, - DS extends NearCustomDatasource = NearCustomDatasource, ->( - processor: - | SecondLayerHandlerProcessor_0_0_0 - | SecondLayerHandlerProcessor_1_0_0, -): processor is SecondLayerHandlerProcessor_0_0_0 { - // Exisiting datasource processors had no concept of specVersion, therefore undefined is equivalent to 0.0.0 - return processor.specVersion === undefined; -} - -export function isSecondLayerHandlerProcessor_1_0_0< - K extends NearHandlerKind, - F extends Record, - E, - DS extends NearCustomDatasource = NearCustomDatasource, ->( - processor: - | SecondLayerHandlerProcessor_0_0_0 - | SecondLayerHandlerProcessor_1_0_0, -): processor is SecondLayerHandlerProcessor_1_0_0 { - return processor.specVersion === '1.0.0'; -} - -export function asSecondLayerHandlerProcessor_1_0_0< - K extends NearHandlerKind, - F extends Record, - E, - DS extends NearCustomDatasource = NearCustomDatasource, ->( - processor: - | SecondLayerHandlerProcessor_0_0_0 - | SecondLayerHandlerProcessor_1_0_0, -): SecondLayerHandlerProcessor_1_0_0 { - if (isSecondLayerHandlerProcessor_1_0_0(processor)) { - return processor; - } - - if (!isSecondLayerHandlerProcessor_0_0_0(processor)) { - throw new Error('Unsupported ds processor version'); - } - - return { - ...processor, - specVersion: '1.0.0', - filterProcessor: (params) => - processor.filterProcessor(params.filter, params.input, params.ds), - transformer: (params) => - processor - .transformer(params.input, params.ds, params.api, params.assets) - .then((res) => [res]), - }; -} +import { NearMapping, NearCustomHandler } from '@subql/types-near'; @Injectable() export class DsProcessorService extends BaseDsProcessorService< diff --git a/packages/node/src/indexer/dynamic-ds.service.ts b/packages/node/src/indexer/dynamic-ds.service.ts index 28c5f67e..effbd952 100644 --- a/packages/node/src/indexer/dynamic-ds.service.ts +++ b/packages/node/src/indexer/dynamic-ds.service.ts @@ -7,37 +7,28 @@ import { DatasourceParams, DynamicDsService as BaseDynamicDsService, } from '@subql/node-core'; -import { cloneDeep } from 'lodash'; import { NearProjectDs, SubqueryProject } from '../configure/SubqueryProject'; import { DsProcessorService } from './ds-processor.service'; @Injectable() -export class DynamicDsService extends BaseDynamicDsService { +export class DynamicDsService extends BaseDynamicDsService< + NearProjectDs, + SubqueryProject +> { constructor( private readonly dsProcessorService: DsProcessorService, - @Inject('ISubqueryProject') private readonly project: SubqueryProject, + @Inject('ISubqueryProject') project: SubqueryProject, ) { - super(); + super(project); } protected async getDatasource( params: DatasourceParams, ): Promise { - const t = this.project.templates.find( - (t) => t.name === params.templateName, + const dsObj = this.getTemplate( + params.templateName, + params.startBlock, ); - if (!t) { - throw new Error( - `Unable to find matching template in project for name: "${params.templateName}"`, - ); - } - const { name, ...template } = cloneDeep(t); - - const dsObj = { - ...template, - startBlock: params.startBlock, - } as NearProjectDs; - try { if (isCustomDs(dsObj)) { dsObj.processor.options = { diff --git a/packages/node/src/indexer/fetch.module.ts b/packages/node/src/indexer/fetch.module.ts index eac70050..246757ad 100644 --- a/packages/node/src/indexer/fetch.module.ts +++ b/packages/node/src/indexer/fetch.module.ts @@ -10,12 +10,12 @@ import { PoiService, NodeConfig, ConnectionPoolService, - SmartBatchService, StoreCacheService, ConnectionPoolStateManager, IProjectUpgradeService, PoiSyncService, InMemoryCacheService, + SandboxService, } from '@subql/node-core'; import { SubqueryProject } from '../configure/SubqueryProject'; import { ApiService } from './api.service'; @@ -30,7 +30,6 @@ import { FetchService } from './fetch.service'; import { IndexerManager } from './indexer.manager'; import { NearApiConnection } from './nearApi.connection'; import { ProjectService } from './project.service'; -import { SandboxService } from './sandbox.service'; import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; @Module({ @@ -42,13 +41,6 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; IndexerManager, ConnectionPoolService, ConnectionPoolStateManager, - { - provide: SmartBatchService, - useFactory: (nodeConfig: NodeConfig) => { - return new SmartBatchService(nodeConfig.batchSize); - }, - inject: [NodeConfig], - }, { provide: 'IBlockDispatcher', useFactory: ( @@ -58,7 +50,6 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; projectUpgradeService: IProjectUpgradeService, apiService: ApiService, indexerManager: IndexerManager, - smartBatchService: SmartBatchService, cacheService: InMemoryCacheService, storeService: StoreService, storeCacheService: StoreCacheService, @@ -74,7 +65,6 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; eventEmitter, projectService, projectUpgradeService, - smartBatchService, cacheService, storeService, storeCacheService, @@ -91,12 +81,10 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; eventEmitter, projectService, projectUpgradeService, - smartBatchService, storeService, storeCacheService, poiSyncService, project, - dynamicDsService, ), inject: [ NodeConfig, @@ -105,7 +93,6 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; 'IProjectUpgradeService', ApiService, IndexerManager, - SmartBatchService, InMemoryCacheService, StoreService, StoreCacheService, diff --git a/packages/node/src/indexer/fetch.service.ts b/packages/node/src/indexer/fetch.service.ts index 59d45447..b1745c8a 100644 --- a/packages/node/src/indexer/fetch.service.ts +++ b/packages/node/src/indexer/fetch.service.ts @@ -18,8 +18,6 @@ import { calcInterval, nearHeaderToHeader } from '../utils/near'; import { ApiService } from './api.service'; import { INearBlockDispatcher } from './blockDispatcher'; import { NearDictionaryService } from './dictionary'; -import { DsProcessorService } from './ds-processor.service'; -import { DynamicDsService } from './dynamic-ds.service'; import { ProjectService } from './project.service'; import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; @@ -40,8 +38,6 @@ export class FetchService extends BaseFetchService< @Inject('IBlockDispatcher') blockDispatcher: INearBlockDispatcher, dictionaryService: NearDictionaryService, - private dsProcessorService: DsProcessorService, - dynamicDsService: DynamicDsService, private unfinalizedBlocksService: UnfinalizedBlocksService, eventEmitter: EventEmitter2, schedulerRegistry: SchedulerRegistry, @@ -61,10 +57,6 @@ export class FetchService extends BaseFetchService< return this.apiService.unsafeApi; } - protected getGenesisHash(): string { - return this.apiService.networkMeta.genesisHash; - } - protected async getFinalizedHeight(): Promise { const finalizedHeader = (await this.api.block({ finality: 'final' })) .header; diff --git a/packages/node/src/indexer/indexer.manager.ts b/packages/node/src/indexer/indexer.manager.ts index 3e973197..1d4dd678 100644 --- a/packages/node/src/indexer/indexer.manager.ts +++ b/packages/node/src/indexer/indexer.manager.ts @@ -19,6 +19,7 @@ import { ProcessBlockResponse, BaseIndexerManager, IBlock, + SandboxService, } from '@subql/node-core'; import { NearBlock, @@ -33,19 +34,15 @@ import { JsonRpcProvider } from 'near-api-js/lib/providers'; import { NearProjectDs } from '../configure/SubqueryProject'; import * as NearUtil from '../utils/near'; import { ApiService, SafeJsonRpcProvider } from './api.service'; -import { - asSecondLayerHandlerProcessor_1_0_0, - DsProcessorService, -} from './ds-processor.service'; +import { DsProcessorService } from './ds-processor.service'; import { DynamicDsService } from './dynamic-ds.service'; -import { SandboxService } from './sandbox.service'; import { BlockContent } from './types'; import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; @Injectable() export class IndexerManager extends BaseIndexerManager< - SafeJsonRpcProvider, JsonRpcProvider, + SafeJsonRpcProvider, BlockContent, ApiService, NearDatasource, @@ -56,12 +53,11 @@ export class IndexerManager extends BaseIndexerManager< > { protected isRuntimeDs = isRuntimeDs; protected isCustomDs = isCustomDs; - protected updateCustomProcessor = asSecondLayerHandlerProcessor_1_0_0; constructor( apiService: ApiService, nodeConfig: NodeConfig, - sandboxService: SandboxService, + sandboxService: SandboxService, dsProcessorService: DsProcessorService, dynamicDsService: DynamicDsService, unfinalizedBlocksService: UnfinalizedBlocksService, @@ -88,14 +84,6 @@ export class IndexerManager extends BaseIndexerManager< ); } - getBlockHeight(block: BlockContent): number { - return block.block.header.height; - } - - getBlockHash(block: BlockContent): string { - return block.block.header.hash; - } - // eslint-disable-next-line @typescript-eslint/require-await private async getApi(block: BlockContent): Promise { return this.apiService.safeApi(block.block.header.height); diff --git a/packages/node/src/indexer/nearApi.connection.ts b/packages/node/src/indexer/nearApi.connection.ts index ba605830..ffcf5506 100644 --- a/packages/node/src/indexer/nearApi.connection.ts +++ b/packages/node/src/indexer/nearApi.connection.ts @@ -33,7 +33,7 @@ export class NearApiConnection { readonly networkMeta: NetworkMetadataPayload; - constructor( + private constructor( public unsafeApi: Near.providers.JsonRpcProvider, private fetchBlocksBatches: FetchFunc, chainId: string, diff --git a/packages/node/src/indexer/project.service.ts b/packages/node/src/indexer/project.service.ts index 7c310cc6..f44a537a 100644 --- a/packages/node/src/indexer/project.service.ts +++ b/packages/node/src/indexer/project.service.ts @@ -40,7 +40,7 @@ export class ProjectService extends BaseProjectService< @Inject(isMainThread ? Sequelize : 'Null') sequelize: Sequelize, @Inject('ISubqueryProject') project: SubqueryProject, @Inject('IProjectUpgradeService') - protected readonly projectUpgradeService: IProjectUpgradeService, + projectUpgradeService: IProjectUpgradeService, @Inject(isMainThread ? StoreService : 'Null') storeService: StoreService, nodeConfig: NodeConfig, dynamicDsService: DynamicDsService, diff --git a/packages/node/src/indexer/sandbox.service.ts b/packages/node/src/indexer/sandbox.service.ts deleted file mode 100644 index ee08751c..00000000 --- a/packages/node/src/indexer/sandbox.service.ts +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import { isMainThread } from 'worker_threads'; -import { Inject, Injectable } from '@nestjs/common'; -import { - hostStoreToStore, - IndexerSandbox, - InMemoryCacheService, - ISubqueryProject, - NodeConfig, - StoreService, -} from '@subql/node-core'; -import { Store, BaseDataSource } from '@subql/types-core'; -import { ApiService } from './api.service'; - -/* It would be nice to move this to node core but need to find a way to inject other things into the sandbox */ -@Injectable() -export class SandboxService { - private processorCache: Record = {}; - - constructor( - private readonly apiService: ApiService, - @Inject(isMainThread ? StoreService : 'Null') - private readonly storeService: StoreService, - private readonly cacheService: InMemoryCacheService, - private readonly nodeConfig: NodeConfig, - @Inject('ISubqueryProject') private readonly project: ISubqueryProject, - ) {} - - getDsProcessor(ds: BaseDataSource, api: Api): IndexerSandbox { - const store: Store = isMainThread - ? this.storeService.getStore() - : hostStoreToStore((global as any).host); // Provided in worker.ts - - const cache = this.cacheService.getCache(); - const entry = this.getDataSourceEntry(ds); - let processor = this.processorCache[entry]; - if (!processor) { - processor = new IndexerSandbox( - { - cache, - store, - root: this.project.root, - entry, - chainId: this.project.network.chainId, - }, - this.nodeConfig, - ); - this.processorCache[entry] = processor; - } - - processor.freeze(api, 'api'); - if (this.nodeConfig.unsafe) { - processor.freeze(this.apiService.api, 'unsafeApi'); - } - processor.freeze(this.project.network.chainId, 'chainId'); - return processor; - } - - private getDataSourceEntry(ds: BaseDataSource): string { - return ds.mapping.file; - } -} diff --git a/packages/node/src/indexer/worker/worker-fetch.module.ts b/packages/node/src/indexer/worker/worker-fetch.module.ts index ce4b10c9..c9fb8d60 100644 --- a/packages/node/src/indexer/worker/worker-fetch.module.ts +++ b/packages/node/src/indexer/worker/worker-fetch.module.ts @@ -11,6 +11,8 @@ import { NodeConfig, InMemoryCacheService, WorkerInMemoryCacheService, + SandboxService, + WorkerUnfinalizedBlocksService, } from '@subql/node-core'; import { SubqueryProject } from '../../configure/SubqueryProject'; import { ApiService } from '../api.service'; @@ -19,10 +21,8 @@ import { DynamicDsService } from '../dynamic-ds.service'; import { IndexerManager } from '../indexer.manager'; import { NearApiConnection } from '../nearApi.connection'; import { ProjectService } from '../project.service'; -import { SandboxService } from '../sandbox.service'; import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service'; import { WorkerService } from '../worker/worker.service'; -import { WorkerUnfinalizedBlocksService } from '../worker/worker.unfinalizedBlocks.service'; @Module({ providers: [ diff --git a/packages/node/src/indexer/worker/worker.unfinalizedBlocks.service.ts b/packages/node/src/indexer/worker/worker.unfinalizedBlocks.service.ts deleted file mode 100644 index 47c2a244..00000000 --- a/packages/node/src/indexer/worker/worker.unfinalizedBlocks.service.ts +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import { isMainThread } from 'worker_threads'; -import { Injectable } from '@nestjs/common'; -import { - IUnfinalizedBlocksService, - Header, - HostUnfinalizedBlocks, - IBlock, -} from '@subql/node-core'; -import { BlockContent } from '../types'; - -@Injectable() -export class WorkerUnfinalizedBlocksService - implements IUnfinalizedBlocksService -{ - constructor(private host: HostUnfinalizedBlocks) { - if (isMainThread) { - throw new Error('Expected to be worker thread'); - } - } - - async processUnfinalizedBlocks( - block: IBlock, - ): Promise { - return this.processUnfinalizedBlockHeader(block.getHeader()); - } - - async processUnfinalizedBlockHeader(header: Header): Promise { - return this.host.unfinalizedBlocksProcess(header); - } - - // eslint-disable-next-line @typescript-eslint/promise-function-async - init(reindex: (targetHeight: number) => Promise): Promise { - throw new Error('This method should not be called from a worker'); - } - resetUnfinalizedBlocks(): void { - throw new Error('This method should not be called from a worker'); - } - resetLastFinalizedVerifiedHeight(): void { - throw new Error('This method should not be called from a worker'); - } - // eslint-disable-next-line @typescript-eslint/promise-function-async - getMetadataUnfinalizedBlocks(): Promise { - throw new Error('This method should not be called from a worker'); - } -} diff --git a/packages/node/src/subcommands/forceClean.init.ts b/packages/node/src/subcommands/forceClean.init.ts index d4b2d7d3..2fadff28 100644 --- a/packages/node/src/subcommands/forceClean.init.ts +++ b/packages/node/src/subcommands/forceClean.init.ts @@ -1,21 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import { NestFactory } from '@nestjs/core'; -import { ForceCleanService, getLogger } from '@subql/node-core'; +import { forceClean } from '@subql/node-core'; import { ForceCleanModule } from './forceClean.module'; -const logger = getLogger('CLI'); -export async function forceCleanInit(): Promise { - try { - const app = await NestFactory.create(ForceCleanModule); - await app.init(); - const forceCleanService = app.get(ForceCleanService); - await forceCleanService.forceClean(); - } catch (e) { - logger.error(e, 'Force-clean failed to execute'); - process.exit(1); - } - - process.exit(0); -} +export const forceCleanInit = (): Promise => forceClean(ForceCleanModule); diff --git a/packages/node/src/subcommands/reindex.init.ts b/packages/node/src/subcommands/reindex.init.ts index 99fbb1ad..2f9fc100 100644 --- a/packages/node/src/subcommands/reindex.init.ts +++ b/packages/node/src/subcommands/reindex.init.ts @@ -1,30 +1,8 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import { NestFactory } from '@nestjs/core'; -import { getLogger, ReindexService } from '@subql/node-core'; +import { reindexInit as baseReindexInit } from '@subql/node-core'; import { ReindexModule } from './reindex.module'; -const logger = getLogger('CLI-Reindex'); -export async function reindexInit(targetHeight: number): Promise { - try { - const app = await NestFactory.create(ReindexModule); - - await app.init(); - const reindexService = app.get(ReindexService); - - await reindexService.init(); - const actualReindexHeight = - await reindexService.getTargetHeightWithUnfinalizedBlocks(targetHeight); - if (actualReindexHeight !== targetHeight) { - logger.info( - `Found index target height ${targetHeight} beyond indexed unfinalized block ${actualReindexHeight}, will index to ${actualReindexHeight}`, - ); - } - await reindexService.reindex(actualReindexHeight); - } catch (e) { - logger.error(e, 'Reindex failed to execute'); - process.exit(1); - } - process.exit(0); -} +export const reindexInit = (targetHeight: number) => + baseReindexInit(ReindexModule, targetHeight); diff --git a/packages/node/src/subcommands/testing.module.ts b/packages/node/src/subcommands/testing.module.ts index 9822c956..bb7220e4 100644 --- a/packages/node/src/subcommands/testing.module.ts +++ b/packages/node/src/subcommands/testing.module.ts @@ -15,6 +15,7 @@ import { StoreCacheService, StoreService, TestRunner, + SandboxService, } from '@subql/node-core'; import { ConfigureModule } from '../configure/configure.module'; import { SubqueryProject } from '../configure/SubqueryProject'; @@ -25,7 +26,6 @@ import { FetchModule } from '../indexer/fetch.module'; import { IndexerManager } from '../indexer/indexer.manager'; import { NearApiConnection } from '../indexer/nearApi.connection'; import { ProjectService } from '../indexer/project.service'; -import { SandboxService } from '../indexer/sandbox.service'; import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service'; import { MetaModule } from '../meta/meta.module'; diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.md index c4c48e07..f88193e4 100644 --- a/packages/types/CHANGELOG.md +++ b/packages/types/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Update dependencies and apply changes to match (#109) ## [3.2.0] - 2024-04-10 ### Changed diff --git a/packages/types/package.json b/packages/types/package.json index 3a2d030c..86cbedab 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -17,7 +17,7 @@ "/dist" ], "dependencies": { - "@subql/types-core": "^0.6.0", + "@subql/types-core": "^0.7.0", "bn.js": "5.2.1", "near-api-js": "^1.1.0" }, diff --git a/packages/types/src/project.ts b/packages/types/src/project.ts index 3f702494..fa6764bf 100644 --- a/packages/types/src/project.ts +++ b/packages/types/src/project.ts @@ -5,13 +5,15 @@ import { BaseTemplateDataSource, IProjectNetworkConfig, CommonSubqueryProject, - DictionaryQueryEntry, FileReference, ProjectManifestV1_0_0, BlockFilter, BaseDataSource, BaseCustomDataSource, BaseHandler, + SecondLayerHandlerProcessor_0_0_0, + SecondLayerHandlerProcessor_1_0_0, + DsProcessor, } from '@subql/types-core'; import {providers} from 'near-api-js'; import {NearBlock, NearTransaction, NearAction, NearTransactionReceipt, ActionType} from './interfaces'; @@ -62,13 +64,6 @@ export type RuntimeHandlerInputMap = { [NearHandlerKind.Receipt]: NearTransactionReceipt; }; -type RuntimeFilterMap = { - [NearHandlerKind.Block]: NearBlockFilter; - [NearHandlerKind.Transaction]: NearTransactionFilter; - [NearHandlerKind.Action]: NearActionFilter; - [NearHandlerKind.Receipt]: NearTransactionFilter; -}; - /** * Represents a filter for Near blocks. */ @@ -248,34 +243,6 @@ export interface NearCustomDatasource { - ( - input: RuntimeHandlerInputMap[T], - ds: DS, - api: providers.JsonRpcProvider, - assets?: Record - ): Promise; // | NearBuiltinDataSource -} - -export interface HandlerInputTransformer_1_0_0< - T extends NearHandlerKind, - F extends Record, - E, - DS extends NearCustomDatasource = NearCustomDatasource -> { - (params: { - input: RuntimeHandlerInputMap[T]; - ds: DS; - filter?: F; - api: providers.JsonRpcProvider; - assets?: Record; - }): Promise; // | NearBuiltinDataSource -} - type SecondLayerHandlerProcessorArray< K extends string, F extends Record, @@ -287,7 +254,7 @@ type SecondLayerHandlerProcessorArray< | SecondLayerHandlerProcessor | SecondLayerHandlerProcessor; -export interface NearDatasourceProcessor< +export type NearDatasourceProcessor< K extends string, F extends Record, DS extends NearCustomDatasource = NearCustomDatasource, @@ -295,53 +262,16 @@ export interface NearDatasourceProcessor< string, SecondLayerHandlerProcessorArray > -> { - kind: K; - validate(ds: DS, assets: Record): void; - dsFilterProcessor(ds: DS, api: providers.JsonRpcProvider): boolean; - handlerProcessors: P; -} - -interface SecondLayerHandlerProcessorBase< - K extends NearHandlerKind, - F extends Record, - DS extends NearCustomDatasource = NearCustomDatasource -> { - baseHandlerKind: K; - baseFilter: RuntimeFilterMap[K] | RuntimeFilterMap[K][]; - filterValidator: (filter?: F) => void; - dictionaryQuery?: (filter: F, ds: DS) => DictionaryQueryEntry | undefined; -} - -// only allow one custom handler for each baseHandler kind -export interface SecondLayerHandlerProcessor_0_0_0< - K extends NearHandlerKind, - F extends Record, - E, - DS extends NearCustomDatasource = NearCustomDatasource -> extends SecondLayerHandlerProcessorBase { - specVersion: undefined; - transformer: HandlerInputTransformer_0_0_0; - filterProcessor: (filter: F | undefined, input: RuntimeHandlerInputMap[K], ds: DS) => boolean; -} - -export interface SecondLayerHandlerProcessor_1_0_0< - K extends NearHandlerKind, - F extends Record, - E, - DS extends NearCustomDatasource = NearCustomDatasource -> extends SecondLayerHandlerProcessorBase { - specVersion: '1.0.0'; - transformer: HandlerInputTransformer_1_0_0; - filterProcessor: (params: {filter: F | undefined; input: RuntimeHandlerInputMap[K]; ds: DS}) => boolean; -} +> = DsProcessor; export type SecondLayerHandlerProcessor< K extends NearHandlerKind, F extends Record, E, DS extends NearCustomDatasource = NearCustomDatasource -> = SecondLayerHandlerProcessor_0_0_0 | SecondLayerHandlerProcessor_1_0_0; +> = + | SecondLayerHandlerProcessor_0_0_0 + | SecondLayerHandlerProcessor_1_0_0; export type NearProject = CommonSubqueryProject< IProjectNetworkConfig, diff --git a/yarn.lock b/yarn.lock index 291b03b5..c5e45020 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2760,7 +2760,7 @@ __metadata: version: 0.0.0-use.local resolution: "@subql/common-near@workspace:packages/common-near" dependencies: - "@subql/common": ^3.5.0 + "@subql/common": ^3.5.1 "@subql/types-near": "workspace:*" "@types/bn.js": 4.11.6 "@types/js-yaml": ^4.0.4 @@ -2773,11 +2773,11 @@ __metadata: languageName: unknown linkType: soft -"@subql/common@npm:3.5.0, @subql/common@npm:^3.5.0": - version: 3.5.0 - resolution: "@subql/common@npm:3.5.0" +"@subql/common@npm:3.5.1, @subql/common@npm:^3.5.1": + version: 3.5.1 + resolution: "@subql/common@npm:3.5.1" dependencies: - "@subql/types-core": 0.6.0 + "@subql/types-core": 0.7.0 axios: ^0.28.0 class-transformer: ^0.5.1 class-validator: ^0.14.0 @@ -2787,22 +2787,22 @@ __metadata: reflect-metadata: ^0.1.13 semver: ^7.5.2 update-notifier: 5.1.0 - checksum: 63c374b82de90f2db5c76d8f3808e37100d3616707736806d3f43249ea0ca63f0c115104815cb842becb0d350e16e893a8d3942c0e865d62890e43362a455799 + checksum: df16b2025635d5096b96c299805d3f9c85f7603b245037b2aae48fc0406e2eedab14af7f9e9b07247f166042854427c68a54d90e83588c8e1850471a14e43a0e languageName: node linkType: hard -"@subql/node-core@npm:^8.0.1": - version: 8.0.1 - resolution: "@subql/node-core@npm:8.0.1" +"@subql/node-core@npm:^10.0.0": + version: 10.0.0 + resolution: "@subql/node-core@npm:10.0.0" dependencies: "@apollo/client": ^3.8.8 "@nestjs/common": ^9.4.0 "@nestjs/event-emitter": ^2.0.0 "@nestjs/schedule": ^3.0.1 - "@subql/common": 3.5.0 - "@subql/testing": 2.1.1-0 - "@subql/types": 3.4.0 - "@subql/utils": 2.9.0 + "@subql/common": 3.5.1 + "@subql/testing": 2.1.1 + "@subql/types": 3.5.0 + "@subql/utils": 2.9.1 "@willsoto/nestjs-prometheus": ^5.4.0 async-lock: ^1.4.0 async-mutex: ^0.4.0 @@ -2814,11 +2814,11 @@ __metadata: merkle-tools: ^1.4.1 prom-client: ^14.0.1 source-map: ^0.7.4 - tar: ^6.1.11 + tar: ^6.2.1 toposort-class: ^1.0.1 vm2: ^3.9.19 yargs: ^16.2.0 - checksum: ead9feeef201b2ed9db1726e70d375d6a3ca5d44f7576dd14aa2949308d11a6a34abcc1013cd4d84eb8e57fd236609cdb93e359c0b85ff5b225257c99261635d + checksum: c52c304ea23c10a171ef411b877032c88733eb6a2262b4a167de9484f7e1fed567bdc36ad1bcb7f9a610d1774a348bd203dc82dd1f7c52f693ff7756c4097d4f languageName: node linkType: hard @@ -2834,9 +2834,9 @@ __metadata: "@nestjs/schedule": ^3.0.1 "@nestjs/schematics": ^9.2.0 "@nestjs/testing": ^9.4.0 - "@subql/common": ^3.5.0 + "@subql/common": ^3.5.1 "@subql/common-near": "workspace:*" - "@subql/node-core": ^8.0.1 + "@subql/node-core": ^10.0.0 "@subql/types-near": "workspace:*" "@types/express": ^4.17.13 "@types/jest": ^27.4.0 @@ -2864,30 +2864,21 @@ __metadata: languageName: unknown linkType: soft -"@subql/testing@npm:2.1.1-0": - version: 2.1.1-0 - resolution: "@subql/testing@npm:2.1.1-0" - dependencies: - "@subql/types-core": ^0.4.1-0 - checksum: d27d1181f2afb1a394a80fc7d7e36bdd1e64981728d41214412cdeab930c45a7d3bd7cdc58d25d4653c3dcdaf2a456a4e7ae67e4f2ccb39f5494905d06cdecad - languageName: node - linkType: hard - -"@subql/types-core@npm:0.6.0, @subql/types-core@npm:^0.6.0": - version: 0.6.0 - resolution: "@subql/types-core@npm:0.6.0" +"@subql/testing@npm:2.1.1": + version: 2.1.1 + resolution: "@subql/testing@npm:2.1.1" dependencies: - package-json-type: ^1.0.3 - checksum: 0688f1973f830d1d2ab34864b00fc7efb105228b0328ac485c69950957da3a67a92d9477bc94945e57e3cccbf1b732bdea097c1a452a907e23704e9cbeff31fc + "@subql/types-core": ^0.7.0 + checksum: 6b81d0130d364984e1a565228523b414c2fd66d128b2795161872657546ea83a7e23b42d59f968241f18c114afd7c511886418db8e89679f8a35861e8c24e5a9 languageName: node linkType: hard -"@subql/types-core@npm:^0.4.1-0": - version: 0.4.1-0 - resolution: "@subql/types-core@npm:0.4.1-0" +"@subql/types-core@npm:0.7.0, @subql/types-core@npm:^0.7.0": + version: 0.7.0 + resolution: "@subql/types-core@npm:0.7.0" dependencies: package-json-type: ^1.0.3 - checksum: f7a51202b6e350e65f65a55136a2b48c8d6ae403850e96215995648656b3ef0779ef33ae526c7ffaa51b221b29255137e84ffd0abb5c1155095b1abfb15ef340 + checksum: ba8d140333d315e80c28b44f29f56511302d0179001048e55ef2bbdded51deb7ddd37d77eee3d6fff1c18f2b944f2cd9f4e39da39e9f4bd9216d4e1b4bbf6378 languageName: node linkType: hard @@ -2895,27 +2886,27 @@ __metadata: version: 0.0.0-use.local resolution: "@subql/types-near@workspace:packages/types" dependencies: - "@subql/types-core": ^0.6.0 + "@subql/types-core": ^0.7.0 "@types/app-module-path": ^2.2.0 bn.js: 5.2.1 near-api-js: ^1.1.0 languageName: unknown linkType: soft -"@subql/types@npm:3.4.0": - version: 3.4.0 - resolution: "@subql/types@npm:3.4.0" +"@subql/types@npm:3.5.0": + version: 3.5.0 + resolution: "@subql/types@npm:3.5.0" dependencies: - "@subql/types-core": 0.6.0 + "@subql/types-core": 0.7.0 peerDependencies: "@polkadot/api": ^10 - checksum: a13e02ebdf35abc05104615f3884faf9f2d9037eece58ac05ba6cc4e07812ef40479645578327147ac82872e9466b5e2995a22076a7fdbc27fd6dd28f1cb669d + checksum: acecfafa56d3488f9ea25a8dd8d3038b1012f090d9f8c8046c74cbf17b983cf19a3d5711a88cb228bc48ce5d5348c9f0dd881a3eb504f319dcbb9b0c6303c526 languageName: node linkType: hard -"@subql/utils@npm:2.9.0": - version: 2.9.0 - resolution: "@subql/utils@npm:2.9.0" +"@subql/utils@npm:2.9.1": + version: 2.9.1 + resolution: "@subql/utils@npm:2.9.1" dependencies: "@polkadot/util": ^12.5.1 "@polkadot/util-crypto": ^12.5.1 @@ -2930,8 +2921,8 @@ __metadata: pino: ^6.13.3 rotating-file-stream: ^3.0.2 semver: ^7.5.2 - tar: ^6.1.11 - checksum: 2536add6833f9a1487ea17eccf0288e48f7a89bb67dfedbc46034902e350d4474be7f38cef06e938ee49ba25274c2271e2b5b6b46e04c965cdba5105f9af4599 + tar: ^6.2.1 + checksum: 555c215f44ac6f6828ade30c3643e260bf99ef3c14381d14a9e9ba3c9fa5cfd0a4e938b3a2fbd907379ad2b832bb79a5071f96966b32d751824affb9cea266c8 languageName: node linkType: hard