sync 20231012 node-core (#76)

subquery · Oct 12, 2023 · ce06c76 · ce06c76
1 parent 2b824a9
commit ce06c76
Show file tree

Hide file tree

Showing 12 changed files with 64 additions and 35 deletions.
diff --git a/packages/common-algorand/CHANGELOG.md b/packages/common-algorand/CHANGELOG.md
@@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+- Version bump with `@subql/common` 3.1.2
+
 ## [3.0.0] - 2023-10-04
 ### Added
 - Parent field to manifest for project upgrades

diff --git a/packages/common-algorand/package.json b/packages/common-algorand/package.json
@@ -13,7 +13,7 @@
   "main": "dist/index.js",
   "license": "GPL-3.0",
   "dependencies": {
-    "@subql/common": "^3.1.1",
+    "@subql/common": "^3.1.2",
     "@subql/types-algorand": "workspace:*",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.13.2",

diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md
@@ -6,6 +6,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+
+### Changed
+- debug has changed from a boolean to a string to allow scoping debug log level (#2077)
+
+### Fixed
+- Sync with node-core.
+  - Fixed Poi migration performance issue.
+  - Fixed AutoQueue timeout issue.
+  - Fixed Poi sync could block DB IO and drop connection issue.
+
+
 ## [3.0.0] - 2023-10-04
 ### Changed
 - Update `@subql/node-core` and sync with main SDK.

diff --git a/packages/node/package.json b/packages/node/package.json
@@ -24,9 +24,9 @@
     "@nestjs/event-emitter": "^2.0.0",
     "@nestjs/platform-express": "^9.4.0",
     "@nestjs/schedule": "^3.0.1",
-    "@subql/common": "^3.1.1",
+    "@subql/common": "^3.1.2",
     "@subql/common-algorand": "workspace:*",
-    "@subql/node-core": "^5.0.3",
+    "@subql/node-core": "^6.0.1",
     "@subql/types-algorand": "workspace:*",
     "algosdk": "^2.2.0",
     "axios": "^1.3.4",

diff --git a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts
@@ -10,6 +10,7 @@ import {
   StoreService,
   IProjectService,
   PoiService,
+  PoiSyncService,
   BlockDispatcher,
   ProcessBlockResponse,
   IProjectUpgradeService,
@@ -44,6 +45,7 @@ export class BlockDispatcherService
     storeService: StoreService,
     storeCacheService: StoreCacheService,
     poiService: PoiService,
+    poiSyncService: PoiSyncService,
     @Inject('ISubqueryProject') project: SubqueryProject,
     dynamicDsService: DynamicDsService,
   ) {
@@ -56,6 +58,7 @@ export class BlockDispatcherService
       storeService,
       storeCacheService,
       poiService,
+      poiSyncService,
       project,
       dynamicDsService,
       async (blockNums: number[]): Promise<AlgorandBlock[]> => {

diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts
@@ -10,6 +10,7 @@ import {
   SmartBatchService,
   StoreService,
   PoiService,
+  PoiSyncService,
   StoreCacheService,
   IProjectService,
   IDynamicDsService,
@@ -92,6 +93,7 @@ export class WorkerBlockDispatcherService
     storeService: StoreService,
     storeCacheService: StoreCacheService,
     poiService: PoiService,
+    poiSyncService: PoiSyncService,
     @Inject('ISubqueryProject') project: SubqueryProject,
     dynamicDsService: DynamicDsService,
     unfinalizedBlocksService: UnfinalizedBlocksService,
@@ -106,6 +108,7 @@ export class WorkerBlockDispatcherService
       storeService,
       storeCacheService,
       poiService,
+      poiSyncService,
       project,
       dynamicDsService,
       () =>

diff --git a/packages/node/src/indexer/fetch.module.ts b/packages/node/src/indexer/fetch.module.ts
@@ -8,6 +8,7 @@ import {
   IndexingBenchmarkService,
   StoreService,
   PoiService,
+  PoiSyncService,
   NodeConfig,
   SmartBatchService,
   ConnectionPoolService,
@@ -75,6 +76,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
         storeService: StoreService,
         storeCacheService: StoreCacheService,
         poiService: PoiService,
+        poiSyncService: PoiSyncService,
         project: SubqueryProject,
         dynamicDsService: DynamicDsService,
         unfinalizedBlocksService: UnfinalizedBlocksService,
@@ -90,6 +92,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
               storeService,
               storeCacheService,
               poiService,
+              poiSyncService,
               project,
               dynamicDsService,
               unfinalizedBlocksService,
@@ -106,6 +109,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
               storeService,
               storeCacheService,
               poiService,
+              poiSyncService,
               project,
               dynamicDsService,
             ),
@@ -120,6 +124,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
         StoreService,
         StoreCacheService,
         PoiService,
+        PoiSyncService,
         'ISubqueryProject',
         DynamicDsService,
         UnfinalizedBlocksService,
@@ -135,6 +140,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
     DsProcessorService,
     DynamicDsService,
     PoiService,
+    PoiSyncService,
     {
       useClass: ProjectService,
       provide: 'IProjectService',

diff --git a/packages/node/src/indexer/project.service.ts b/packages/node/src/indexer/project.service.ts
@@ -6,6 +6,7 @@ import { Inject, Injectable } from '@nestjs/common';
 import { EventEmitter2 } from '@nestjs/event-emitter';
 import {
   PoiService,
+  PoiSyncService,
   BaseProjectService,
   StoreService,
   NodeConfig,
@@ -35,6 +36,8 @@ export class ProjectService extends BaseProjectService<
     dsProcessorService: DsProcessorService,
     apiService: AlgorandApiService,
     @Inject(isMainThread ? PoiService : 'Null') poiService: PoiService,
+    @Inject(isMainThread ? PoiSyncService : 'Null')
+    poiSyncService: PoiSyncService,
     @Inject(isMainThread ? Sequelize : 'Null') sequelize: Sequelize,
     @Inject('ISubqueryProject') project: SubqueryProject,
     @Inject('IProjectUpgradeService')
@@ -49,6 +52,7 @@ export class ProjectService extends BaseProjectService<
       dsProcessorService,
       apiService,
       poiService,
+      poiSyncService,
       sequelize,
       project,
       projectUpgradeService,

diff --git a/packages/node/src/indexer/worker/worker.ts b/packages/node/src/indexer/worker/worker.ts
@@ -34,7 +34,7 @@ const logger = getLogger(`worker #${threadId}`);
 async function initWorker(startHeight: number): Promise<void> {
   try {
     const app = await NestFactory.create(WorkerModule, {
-      logger: new NestLogger(argv.debug), // TIP: If the worker is crashing comment out this line for better logging
+      logger: new NestLogger(!!argv.debug), // TIP: If the worker is crashing comment out this line for better logging
     });
 
     await app.init();

diff --git a/packages/node/src/init.ts b/packages/node/src/init.ts
@@ -16,7 +16,6 @@ const logger = getLogger('subql-node');
 
 export async function bootstrap(): Promise<void> {
   logger.info(`Current ${pjson.name} version is ${pjson.version}`);
-  const debug = argv.debug;
 
   const validate = (x: any) => {
     const p = parseInt(x);
@@ -41,7 +40,7 @@ export async function bootstrap(): Promise<void> {
 
   try {
     const app = await NestFactory.create(AppModule, {
-      logger: new NestLogger(debug),
+      logger: new NestLogger(!!argv.debug),
     });
     await app.init();
 

diff --git a/packages/node/src/yargs.ts b/packages/node/src/yargs.ts
@@ -13,7 +13,7 @@ export const yargsOptions = yargs(hideBin(process.argv))
     builder: {},
     handler: (argv) => {
       initLogger(
-        argv.debug as boolean,
+        argv.debug as string,
         argv.outputFmt as 'json' | 'colored',
         argv.logLevel as string | undefined,
       );
@@ -30,7 +30,7 @@ export const yargsOptions = yargs(hideBin(process.argv))
     builder: {},
     handler: (argv) => {
       initLogger(
-        argv.debug as boolean,
+        argv.debug as string,
         argv.outputFmt as 'json' | 'colored',
         argv.logLevel as string | undefined,
       );
@@ -53,7 +53,7 @@ export const yargsOptions = yargs(hideBin(process.argv))
       }),
     handler: (argv) => {
       initLogger(
-        argv.debug as boolean,
+        argv.debug as string,
         argv.outputFmt as 'json' | 'colored',
         argv.logLevel as string | undefined,
       );
@@ -82,10 +82,8 @@ export const yargsOptions = yargs(hideBin(process.argv))
     },
     debug: {
       demandOption: false,
-      describe:
-        'Show debug information to console output. will forcefully set log level to debug',
-      type: 'boolean',
-      default: false,
+      describe: `Enable debug logging for specific scopes, this will override log-level. "*" will enable debug everywhere, or comma separated strings for specific scopes. e.g. "SQL,dictionary"`,
+      type: 'string',
     },
     'dictionary-resolver': {
       demandOption: false,

diff --git a/yarn.lock b/yarn.lock
@@ -2916,7 +2916,7 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@subql/common-algorand@workspace:packages/common-algorand"
   dependencies:
-    "@subql/common": ^3.1.1
+    "@subql/common": ^3.1.2
     "@subql/types-algorand": "workspace:*"
     "@types/bn.js": 4.11.6
     "@types/js-yaml": ^4.0.4
@@ -2935,9 +2935,9 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@subql/common@npm:3.1.1, @subql/common@npm:^3.1.1":
-  version: 3.1.1
-  resolution: "@subql/common@npm:3.1.1"
+"@subql/common@npm:3.1.2, @subql/common@npm:^3.1.2":
+  version: 3.1.2
+  resolution: "@subql/common@npm:3.1.2"
   dependencies:
     "@subql/types-core": 0.1.1
     axios: ^0.27.2
@@ -2949,7 +2949,7 @@ __metadata:
     reflect-metadata: ^0.1.13
     semver: ^7.5.2
     update-notifier: 5.1.0
-  checksum: 1d1e3070f0a2f2368a7ba24f9b98012ca88362d51e7e53ed146026aa8c2844e8d58335496a19341facf4422aa2cac89b05e87860b0a4f73e542ca91f8240cdd0
+  checksum: e979fac54089276de67a3a498bdf53396ac3726542d9f38350bced06f45921ab937a885a9c6cf183f3f5e7c82877c8ce9e27a5a5bf1f6fae6113baf68b4b3e55
   languageName: node
   linkType: hard
 
@@ -2965,9 +2965,9 @@ __metadata:
     "@nestjs/schedule": ^3.0.1
     "@nestjs/schematics": ^9.2.0
     "@nestjs/testing": ^9.4.0
-    "@subql/common": ^3.1.1
+    "@subql/common": ^3.1.2
     "@subql/common-algorand": "workspace:*"
-    "@subql/node-core": ^5.0.3
+    "@subql/node-core": ^6.0.1
     "@subql/types-algorand": "workspace:*"
     "@types/express": ^4.17.13
     "@types/jest": ^27.4.0
@@ -2996,19 +2996,19 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@subql/node-core@npm:^5.0.3":
-  version: 5.0.3
-  resolution: "@subql/node-core@npm:5.0.3"
+"@subql/node-core@npm:^6.0.1":
+  version: 6.0.1
+  resolution: "@subql/node-core@npm:6.0.1"
   dependencies:
     "@apollo/client": ^3.7.16
     "@nestjs/common": ^9.4.0
     "@nestjs/event-emitter": ^2.0.0
     "@nestjs/schedule": ^3.0.1
     "@subql/apollo-links": ^1.0.2
-    "@subql/common": 3.1.1
+    "@subql/common": 3.1.2
     "@subql/testing": 2.0.3-0
-    "@subql/types": 3.1.1
-    "@subql/utils": 2.4.4-1
+    "@subql/types": 3.1.3-0
+    "@subql/utils": 2.4.4
     "@subql/x-sequelize": 6.32.0-0.0.2
     "@willsoto/nestjs-prometheus": ^5.4.0
     async-lock: ^1.4.0
@@ -3023,7 +3023,7 @@ __metadata:
     tar: ^6.1.11
     vm2: ^3.9.19
     yargs: ^16.2.0
-  checksum: 8c620eb056436da207f079fd38fe34a599022417ca174187ee26e6b65e9442ea917ea1118920edbad5eb6dce88820124c1c5492333164e9b4c8edfba488d0154
+  checksum: 8c118db94ce2756643092ff71b15eb30ec46256cd6bbbd2beefc8f2cae53d7800f9f4d97d2a045d68a7621040c6399da9902717ed25dddd5fea3334d2f2cdbd0
   languageName: node
   linkType: hard
 
@@ -3066,18 +3066,20 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@subql/types@npm:3.1.1":
-  version: 3.1.1
-  resolution: "@subql/types@npm:3.1.1"
+"@subql/types@npm:3.1.3-0":
+  version: 3.1.3-0
+  resolution: "@subql/types@npm:3.1.3-0"
+  dependencies:
+    "@subql/types-core": 0.1.1
   peerDependencies:
     "@polkadot/api": ^10
-  checksum: 0b1765dfea7e180366cb846b6e8c398bb30be762d79823508c0e4a2953be6785316858f0efe67469cbb693f0bf3db0a81c625b7265cdcc095bd449df75e60d63
+  checksum: 80f5d4b0e509fc7824149be027c387cee64c68f3edbda359e3bd9da1f5372b5d512eafda438efd2e26f3aed3b4b1e3e4060069b3c41b56e31f2cc9ddb45787f6
   languageName: node
   linkType: hard
 
-"@subql/utils@npm:2.4.4-1":
-  version: 2.4.4-1
-  resolution: "@subql/utils@npm:2.4.4-1"
+"@subql/utils@npm:2.4.4":
+  version: 2.4.4
+  resolution: "@subql/utils@npm:2.4.4"
   dependencies:
     "@polkadot/util": ^12.2.1
     "@polkadot/util-crypto": ^12.2.1
@@ -3094,7 +3096,7 @@ __metadata:
     rotating-file-stream: ^3.0.2
     semver: ^7.5.2
     tar: ^6.1.11
-  checksum: f80fac522d9739a493b8afe8f69029996b1e597241a60c8e2641a6192da1a03300c76738f9bcdaedc973f208ed60c6e9608483e60dd8711b0d0843d42b647b42
+  checksum: 1b0a4a4435bddc2bb058283a4417611be5b8c0248a99dc9f5ea32a1255b60beb64217e49e6bf1c99fef14625e9fa561bdf5a481e3208229093cdca8dfd59a71f
   languageName: node
   linkType: hard