From 8b5afdb8d16631ff04b827432c49d195725647b1 Mon Sep 17 00:00:00 2001 From: nigiri <168690269+0xnigir1@users.noreply.github.com> Date: Mon, 4 Nov 2024 18:30:04 -0300 Subject: [PATCH] refactor: cleanup code and update docs --- apps/processor/README.md | 38 +++++++++---- apps/processor/package.json | 3 +- apps/processor/src/index.ts | 19 ++++--- apps/processor/src/services/index.ts | 2 + apps/processor/src/services/logger.service.ts | 19 ------- .../src/services/processor.service.ts | 53 +++++++++++++++---- ...rvice.ts => sharedDependencies.service.ts} | 24 ++++----- packages/shared/src/external.ts | 3 +- 8 files changed, 102 insertions(+), 59 deletions(-) create mode 100644 apps/processor/src/services/index.ts delete mode 100644 apps/processor/src/services/logger.service.ts rename apps/processor/src/services/{dependencies.service.ts => sharedDependencies.service.ts} (82%) diff --git a/apps/processor/README.md b/apps/processor/README.md index c9a86af..39e164b 100644 --- a/apps/processor/README.md +++ b/apps/processor/README.md @@ -1,16 +1,19 @@ -# TODO: update README +# Processor Service -> Note: use this app as reference but preferred way is to re-write app -> from zero instead of refactoring this one. -> When you don't need this anymore, you can delete it +This service is the main application that runs the core processing pipeline: -Sample app that uses [sample-lib](../../packages/sample-lib) Blockchain -provider to fetch Vitalik and Zero address native balance and sums them +- Instantiates and coordinates components from [Grants Stack Indexer packages](../../packages/) +- Creates and manages an Orchestrator per chain to process blockchain events + +## Requirements + +- A running instance of PostgreSQL Data Layer with migrations applied +- A running instance of Envio Indexer ## Setup -1. Change package name to your own in [`package.json`](./package.json) -2. Install dependencies running `pnpm install` +1. Install dependencies running `pnpm install` +2. Build the app using `pnpm build` ### ⚙️ Setting up env variables @@ -23,7 +26,17 @@ $ cp .env.example .env Available options: | Name | Description | Default | Required | Notes | |-----------------------------|--------------------------------------------------------------------------------------------------------------------------------|-----------|----------------------------------|-----------------------------------------------------------------| -| `RPC_URL` | RPC URL to use for querying balances | N/A | Yes | | +| `RPC_URLS` | Array of RPC URLs | N/A | Yes | Multiple URLs for redundancy | +| `CHAIN_ID` | Chain ID | N/A | Yes | At the moment only Optimism is supported (10) | +| `FETCH_LIMIT` | Maximum number of items to fetch in one batch | 500 | Yes | | +| `FETCH_DELAY_MS` | Delay between fetch operations in milliseconds | 3000 | Yes | | +| `DATABASE_URL` | PostgreSQL Data Layer database connection URL | N/A | Yes | | +| `DATABASE_SCHEMA` | PostgreSQL Data Layer database schema name | chainDataSchema | Yes | | +| `INDEXER_GRAPHQL_URL` | GraphQL endpoint for the indexer | N/A | Yes | | +| `INDEXER_ADMIN_SECRET` | Admin secret for indexer authentication | N/A | Yes | | +| `IPFS_GATEWAYS_URL` | Array of IPFS gateway URLs | N/A | Yes | Multiple gateways for redundancy | +| `COINGECKO_API_KEY` | API key for CoinGecko service | N/A | Yes | | +| `COINGECKO_API_TYPE` | CoinGecko API tier (demo or pro) | N/A | Yes | | ## Available Scripts @@ -34,10 +47,15 @@ Available scripts that can be run using `pnpm`: | `build` | Build library using tsc | | `check-types` | Check types issues using tsc | | `clean` | Remove `dist` folder | +| `dev` | Run the app in development mode using tsx | +| `dev:watch` | Run the app in watch mode for development | | `lint` | Run ESLint to check for coding standards | | `lint:fix` | Run linter and automatically fix code formatting issues | | `format` | Check code formatting and style using Prettier | | `format:fix` | Run formatter and automatically fix issues | -| `start` | Run the app | +| `start` | Run the compiled app from dist folder | | `test` | Run tests using vitest | | `test:cov` | Run tests with coverage report | + +TODO: e2e tests +TODO: Docker image diff --git a/apps/processor/package.json b/apps/processor/package.json index 3e81c26..4b280e7 100644 --- a/apps/processor/package.json +++ b/apps/processor/package.json @@ -7,7 +7,8 @@ "build": "tsc -p tsconfig.build.json", "check-types": "tsc --noEmit -p ./tsconfig.json", "clean": "rm -rf dist", - "dev": "tsx watch src/index.ts", + "dev": "tsx src/index.ts", + "dev:watch": "tsx watch src/index.ts", "format": "prettier --check \"{src,test}/**/*.{js,ts,json}\"", "format:fix": "prettier --write \"{src,test}/**/*.{js,ts,json}\"", "lint": "eslint \"{src,test}/**/*.{js,ts,json}\"", diff --git a/apps/processor/src/index.ts b/apps/processor/src/index.ts index 4f29280..11c4bd4 100644 --- a/apps/processor/src/index.ts +++ b/apps/processor/src/index.ts @@ -3,12 +3,13 @@ import { inspect } from "util"; import { environment } from "./config/index.js"; import { ProcessorService } from "./services/processor.service.js"; +let processor: ProcessorService; + const main = async (): Promise => { - const processor = new ProcessorService(environment); + processor = new ProcessorService(environment); await processor.start(); }; -// Handle uncaught errors process.on("unhandledRejection", (reason, p) => { console.error(`Unhandled Rejection at: \n${inspect(p, undefined, 100)}, \nreason: ${reason}`); process.exit(1); @@ -21,8 +22,12 @@ process.on("uncaughtException", (error: Error) => { process.exit(1); }); -// Start the application -main().catch((err) => { - console.error(`Caught error in main handler: ${err}`); - process.exit(1); -}); +main() + .catch((err) => { + console.error(`Caught error in main handler: ${err}`); + process.exit(1); + }) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .finally(async () => { + await processor?.releaseResources(); + }); diff --git a/apps/processor/src/services/index.ts b/apps/processor/src/services/index.ts new file mode 100644 index 0000000..4164fad --- /dev/null +++ b/apps/processor/src/services/index.ts @@ -0,0 +1,2 @@ +export * from "./sharedDependencies.service.js"; +export * from "./processor.service.js"; diff --git a/apps/processor/src/services/logger.service.ts b/apps/processor/src/services/logger.service.ts deleted file mode 100644 index 6c599c6..0000000 --- a/apps/processor/src/services/logger.service.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { ILogger } from "@grants-stack-indexer/shared"; - -export class Logger implements ILogger { - info(message: string): void { - console.log(`[INFO] ${message}`); - } - - warn(message: string): void { - console.warn(`[WARN] ${message}`); - } - - error(message: Error | string): void { - console.error(`[ERROR] ${message}`); - } - - debug(message: string): void { - console.debug(`[DEBUG] ${message}`); - } -} diff --git a/apps/processor/src/services/processor.service.ts b/apps/processor/src/services/processor.service.ts index 9d53e13..fd3dd86 100644 --- a/apps/processor/src/services/processor.service.ts +++ b/apps/processor/src/services/processor.service.ts @@ -1,23 +1,40 @@ +import { optimism } from "viem/chains"; + +import { EvmProvider } from "@grants-stack-indexer/chain-providers"; import { Orchestrator } from "@grants-stack-indexer/data-flow"; -import { ChainId } from "@grants-stack-indexer/shared"; +import { ChainId, Logger } from "@grants-stack-indexer/shared"; import { Environment } from "../config/env.js"; -import { DependenciesService } from "./dependencies.service.js"; -import { Logger } from "./logger.service.js"; +import { SharedDependencies, SharedDependenciesService } from "./index.js"; +/** + * Processor service application + * - Initializes core dependencies (repositories, providers) via SharedDependenciesService + * - Sets up EVM provider with configured RPC endpoints + * - Creates an Orchestrator instance to coordinate an specific chain: + * - Fetching on-chain events via indexer client + * - Processing events through registered handlers + * - Storing processed data in PostgreSQL via repositories + * - Manages graceful shutdown on termination signals + * + * TODO: support multichain + */ export class ProcessorService { - private readonly logger = new Logger(); + private readonly logger = Logger.getInstance(); private readonly orchestrator: Orchestrator; + private readonly kyselyDatabase: SharedDependencies["kyselyDatabase"]; constructor(private readonly env: Environment) { - const { core, registries, indexerClient } = DependenciesService.initialize( - env, - this.logger, - ); + const { core, registries, indexerClient, kyselyDatabase } = + SharedDependenciesService.initialize(env); + this.kyselyDatabase = kyselyDatabase; + + // Initialize EVM provider + const evmProvider = new EvmProvider(env.RPC_URLS, optimism, this.logger); this.orchestrator = new Orchestrator( env.CHAIN_ID as ChainId, - core, + { ...core, evmProvider }, indexerClient, registries, env.FETCH_LIMIT, @@ -25,6 +42,11 @@ export class ProcessorService { ); } + /** + * Start the processor service + * + * The processor runs indefinitely until it is terminated. + */ async start(): Promise { this.logger.info("Starting processor service..."); @@ -48,4 +70,17 @@ export class ProcessorService { throw error; } } + + /** + * Call this function when the processor service is terminated + * - Releases database resources + */ + async releaseResources(): Promise { + try { + this.logger.info("Releasing resources..."); + await this.kyselyDatabase.destroy(); + } catch (error) { + this.logger.error(`Error releasing resources: ${error}`); + } + } } diff --git a/apps/processor/src/services/dependencies.service.ts b/apps/processor/src/services/sharedDependencies.service.ts similarity index 82% rename from apps/processor/src/services/dependencies.service.ts rename to apps/processor/src/services/sharedDependencies.service.ts index 52a0478..7ca798c 100644 --- a/apps/processor/src/services/dependencies.service.ts +++ b/apps/processor/src/services/sharedDependencies.service.ts @@ -1,6 +1,3 @@ -import { optimism } from "viem/chains"; - -import { EvmProvider } from "@grants-stack-indexer/chain-providers"; import { CoreDependencies, InMemoryEventsRegistry, @@ -15,24 +12,27 @@ import { KyselyProjectRepository, KyselyRoundRepository, } from "@grants-stack-indexer/repository"; -import { ILogger } from "@grants-stack-indexer/shared"; import { Environment } from "../config/index.js"; -export type Dependencies = { - core: CoreDependencies; +export type SharedDependencies = { + core: Omit; registries: { eventsRegistry: InMemoryEventsRegistry; strategyRegistry: InMemoryStrategyRegistry; }; indexerClient: EnvioIndexerClient; + kyselyDatabase: ReturnType; }; -export class DependenciesService { - static initialize(env: Environment, logger: ILogger): Dependencies { - // Initialize EVM provider - const evmProvider = new EvmProvider(env.RPC_URLS, optimism, logger); - +/** + * Shared dependencies service + * - Initializes core dependencies (repositories, providers) + * - Initializes registries + * - Initializes indexer client + */ +export class SharedDependenciesService { + static initialize(env: Environment): SharedDependencies { // Initialize repositories const kyselyDatabase = createKyselyDatabase({ connectionString: env.DATABASE_URL, @@ -63,7 +63,6 @@ export class DependenciesService { return { core: { - evmProvider, projectRepository, roundRepository, applicationRepository, @@ -75,6 +74,7 @@ export class DependenciesService { strategyRegistry, }, indexerClient, + kyselyDatabase, }; } } diff --git a/packages/shared/src/external.ts b/packages/shared/src/external.ts index c3ed0c1..7d7bcd8 100644 --- a/packages/shared/src/external.ts +++ b/packages/shared/src/external.ts @@ -10,7 +10,8 @@ export { export type { DeepPartial } from "./utils/testing.js"; export { mergeDeep } from "./utils/testing.js"; -export type { ILogger, Logger } from "./internal.js"; +export type { ILogger } from "./logger/logger.interface.js"; +export { Logger } from "./logger/logger.js"; export { BigNumber } from "./internal.js"; export type { BigNumberType } from "./internal.js";