diff --git a/package-lock.json b/package-lock.json index bf83baeca7..c4b8ffb039 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5503,13 +5503,13 @@ "dev": true }, "node_modules/@nestjs/axios": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@nestjs/axios/-/axios-3.0.0.tgz", - "integrity": "sha512-ULdH03jDWkS5dy9X69XbUVbhC+0pVnrRcj7bIK/ytTZ76w7CgvTZDJqsIyisg3kNOiljRW/4NIjSf3j6YGvl+g==", + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@nestjs/axios/-/axios-3.1.3.tgz", + "integrity": "sha512-RZ/63c1tMxGLqyG3iOCVt7A72oy4x1eM6QEhd4KzCYpaVWW0igq0WSREeRoEZhIxRcZfDfIIkvsOMiM7yfVGZQ==", + "license": "MIT", "peerDependencies": { "@nestjs/common": "^7.0.0 || ^8.0.0 || ^9.0.0 || ^10.0.0", "axios": "^1.3.1", - "reflect-metadata": "^0.1.12", "rxjs": "^6.0.0 || ^7.0.0" } }, @@ -19048,11 +19048,12 @@ } }, "node_modules/axios": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.1.tgz", - "integrity": "sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==", + "version": "1.7.9", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.9.tgz", + "integrity": "sha512-LhLcE7Hbiryz8oMDdDptSrWowmB4Bl6RCt6sIJKpRB4XtVf0iEgewX3au/pJqm+Py1kCASkb/FFKjxQaLtxJvw==", + "license": "MIT", "dependencies": { - "follow-redirects": "^1.15.0", + "follow-redirects": "^1.15.6", "form-data": "^4.0.0", "proxy-from-env": "^1.1.0" } @@ -26015,15 +26016,16 @@ "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" }, "node_modules/follow-redirects": { - "version": "1.15.3", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", - "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==", + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", "funding": [ { "type": "individual", "url": "https://github.com/sponsors/RubenVerborgh" } ], + "license": "MIT", "engines": { "node": ">=4.0" }, @@ -54387,12 +54389,14 @@ "version": "0.0.0", "license": "MIT", "dependencies": { + "@nestjs/axios": "^3.1.3", "@nestjs/common": "^9.0.0", "@nestjs/config": "^2.2.0", "@nestjs/core": "^9.0.0", "@nestjs/platform-express": "^9.0.0", "@nestjs/terminus": "^9.1.2", "@willsoto/nestjs-prometheus": "^4.7.0", + "axios": "^1.7.9", "ethers": "6.13.4", "nest-winston": "^1.7.0", "prom-client": "^14.1.0", diff --git a/packages/data-fetcher/.env.example b/packages/data-fetcher/.env.example index cddefc6c7b..7371b2d632 100644 --- a/packages/data-fetcher/.env.example +++ b/packages/data-fetcher/.env.example @@ -16,4 +16,6 @@ RPC_BATCH_MAX_COUNT=10 RPC_BATCH_MAX_SIZE_BYTES=1048576 RPC_BATCH_STALL_TIME_MS=0 -MAX_BLOCKS_BATCH_SIZE=20 \ No newline at end of file +MAX_BLOCKS_BATCH_SIZE=20 + +RPC_HEALTH_CHECK_TIMEOUT_MS=20_000 \ No newline at end of file diff --git a/packages/data-fetcher/package.json b/packages/data-fetcher/package.json index 6c0ce6cb5a..8a0afc8981 100644 --- a/packages/data-fetcher/package.json +++ b/packages/data-fetcher/package.json @@ -24,12 +24,14 @@ "test:e2e": "jest --config ./test/jest-e2e.json" }, "dependencies": { + "@nestjs/axios": "^3.1.3", "@nestjs/common": "^9.0.0", "@nestjs/config": "^2.2.0", "@nestjs/core": "^9.0.0", "@nestjs/platform-express": "^9.0.0", "@nestjs/terminus": "^9.1.2", "@willsoto/nestjs-prometheus": "^4.7.0", + "axios": "^1.7.9", "ethers": "6.13.4", "nest-winston": "^1.7.0", "prom-client": "^14.1.0", diff --git a/packages/data-fetcher/src/config.spec.ts b/packages/data-fetcher/src/config.spec.ts index aae8bc6c74..23b33571ca 100644 --- a/packages/data-fetcher/src/config.spec.ts +++ b/packages/data-fetcher/src/config.spec.ts @@ -22,6 +22,9 @@ describe("config", () => { }, maxBlocksBatchSize: 20, gracefulShutdownTimeoutMs: 0, + healthChecks: { + rpcHealthCheckTimeoutMs: 20_000, + }, }; }); diff --git a/packages/data-fetcher/src/config.ts b/packages/data-fetcher/src/config.ts index c5d1ef86ce..62dd4962f5 100644 --- a/packages/data-fetcher/src/config.ts +++ b/packages/data-fetcher/src/config.ts @@ -15,6 +15,7 @@ export default () => { RPC_BATCH_STALL_TIME_MS, MAX_BLOCKS_BATCH_SIZE, GRACEFUL_SHUTDOWN_TIMEOUT_MS, + RPC_HEALTH_CHECK_TIMEOUT_MS, } = process.env; return { @@ -42,5 +43,8 @@ export default () => { }, maxBlocksBatchSize: parseInt(MAX_BLOCKS_BATCH_SIZE, 10) || 20, gracefulShutdownTimeoutMs: parseInt(GRACEFUL_SHUTDOWN_TIMEOUT_MS, 10) || 0, + healthChecks: { + rpcHealthCheckTimeoutMs: parseInt(RPC_HEALTH_CHECK_TIMEOUT_MS, 10) || 20_000, + }, }; }; diff --git a/packages/data-fetcher/src/health/health.module.ts b/packages/data-fetcher/src/health/health.module.ts index dae128825d..1c4b98ad8c 100644 --- a/packages/data-fetcher/src/health/health.module.ts +++ b/packages/data-fetcher/src/health/health.module.ts @@ -1,11 +1,12 @@ import { Module } from "@nestjs/common"; import { TerminusModule } from "@nestjs/terminus"; +import { HttpModule } from "@nestjs/axios"; import { HealthController } from "./health.controller"; import { JsonRpcHealthIndicator } from "./jsonRpcProvider.health"; @Module({ controllers: [HealthController], - imports: [TerminusModule], + imports: [TerminusModule, HttpModule], providers: [JsonRpcHealthIndicator], }) export class HealthModule {} diff --git a/packages/data-fetcher/src/health/jsonRpcProvider.health.spec.ts b/packages/data-fetcher/src/health/jsonRpcProvider.health.spec.ts index 2cfaa28919..e495025bbb 100644 --- a/packages/data-fetcher/src/health/jsonRpcProvider.health.spec.ts +++ b/packages/data-fetcher/src/health/jsonRpcProvider.health.spec.ts @@ -1,17 +1,20 @@ import { Test, TestingModule } from "@nestjs/testing"; +import { Logger } from "@nestjs/common"; import { mock } from "jest-mock-extended"; -import { HealthCheckError } from "@nestjs/terminus"; import { JsonRpcProviderBase } from "../rpcProvider"; import { JsonRpcHealthIndicator } from "./jsonRpcProvider.health"; +import { ConfigService } from "@nestjs/config"; +import { HttpService } from "@nestjs/axios"; +import { of, throwError } from "rxjs"; +import { AxiosError } from "axios"; describe("JsonRpcHealthIndicator", () => { - const healthIndicatorKey = "rpcProvider"; let jsonRpcProviderMock: JsonRpcProviderBase; let jsonRpcHealthIndicator: JsonRpcHealthIndicator; + let httpService: HttpService; + let configService: ConfigService; - beforeEach(async () => { - jsonRpcProviderMock = mock(); - + const getHealthIndicator = async () => { const app: TestingModule = await Test.createTestingModule({ providers: [ JsonRpcHealthIndicator, @@ -19,38 +22,90 @@ describe("JsonRpcHealthIndicator", () => { provide: JsonRpcProviderBase, useValue: jsonRpcProviderMock, }, + { + provide: HttpService, + useValue: httpService, + }, + { + provide: ConfigService, + useValue: configService, + }, ], }).compile(); - jsonRpcHealthIndicator = app.get(JsonRpcHealthIndicator); + app.useLogger(mock()); + return app.get(JsonRpcHealthIndicator); + }; + + beforeEach(async () => { + jsonRpcProviderMock = mock(); + + httpService = mock({ + post: jest.fn(), + }); + + configService = mock({ + get: jest.fn().mockImplementation((key: string) => { + if (key === "blockchain.rpcUrl") return "http://localhost:3050"; + if (key === "healthChecks.rpcHealthCheckTimeoutMs") return 5000; + return null; + }), + }); + + jsonRpcHealthIndicator = await getHealthIndicator(); }); describe("isHealthy", () => { - describe("when rpcProvider is open", () => { - beforeEach(() => { - jest.spyOn(jsonRpcProviderMock, "getState").mockReturnValueOnce("open"); - }); + const rpcRequest = { + id: 1, + jsonrpc: "2.0", + method: "eth_chainId", + params: [], + }; - it("returns OK health indicator result", async () => { - const result = await jsonRpcHealthIndicator.isHealthy(healthIndicatorKey); - expect(result).toEqual({ [healthIndicatorKey]: { rpcProviderState: "open", status: "up" } }); + it("returns healthy status when RPC responds successfully", async () => { + (httpService.post as jest.Mock).mockReturnValueOnce(of({ data: { result: "0x1" } })); + const result = await jsonRpcHealthIndicator.isHealthy("jsonRpcProvider"); + expect(result).toEqual({ + jsonRpcProvider: { + status: "up", + }, }); + expect(httpService.post).toHaveBeenCalledWith("http://localhost:3050", rpcRequest, { timeout: 5000 }); }); - describe("when rpcProvider is closed", () => { - beforeEach(() => { - jest.spyOn(jsonRpcProviderMock, "getState").mockReturnValueOnce("closed"); - }); + it("throws HealthCheckError when RPC request fails", async () => { + const error = new AxiosError(); + error.response = { + status: 503, + data: "Service Unavailable", + } as any; - it("throws HealthCheckError error", async () => { - expect.assertions(2); - try { - await jsonRpcHealthIndicator.isHealthy(healthIndicatorKey); - } catch (error) { - expect(error).toBeInstanceOf(HealthCheckError); - expect(error.message).toBe("JSON RPC provider is not in open state"); - } + (httpService.post as jest.Mock).mockReturnValueOnce(throwError(() => error)); + await expect(jsonRpcHealthIndicator.isHealthy("jsonRpcProvider")).rejects.toThrow(); + expect(httpService.post).toHaveBeenCalledWith("http://localhost:3050", rpcRequest, { timeout: 5000 }); + }); + + it("throws HealthCheckError when RPC request times out", async () => { + const error = new AxiosError(); + error.code = "ECONNABORTED"; + + (httpService.post as jest.Mock).mockReturnValueOnce(throwError(() => error)); + await expect(jsonRpcHealthIndicator.isHealthy("jsonRpcProvider")).rejects.toThrow(); + expect(httpService.post).toHaveBeenCalledWith("http://localhost:3050", rpcRequest, { timeout: 5000 }); + }); + + it("uses configured timeout from config service", async () => { + (configService.get as jest.Mock).mockImplementation((key: string) => { + if (key === "blockchain.rpcUrl") return "http://localhost:3050"; + if (key === "healthChecks.rpcHealthCheckTimeoutMs") return 10000; + return null; }); + jsonRpcHealthIndicator = await getHealthIndicator(); + + (httpService.post as jest.Mock).mockReturnValueOnce(of({ data: { result: "0x1" } })); + await jsonRpcHealthIndicator.isHealthy("jsonRpcProvider"); + expect(httpService.post).toHaveBeenCalledWith("http://localhost:3050", rpcRequest, { timeout: 10000 }); }); }); }); diff --git a/packages/data-fetcher/src/health/jsonRpcProvider.health.ts b/packages/data-fetcher/src/health/jsonRpcProvider.health.ts index 60406bfb4c..e5434382b2 100644 --- a/packages/data-fetcher/src/health/jsonRpcProvider.health.ts +++ b/packages/data-fetcher/src/health/jsonRpcProvider.health.ts @@ -1,22 +1,64 @@ import { Injectable } from "@nestjs/common"; import { HealthIndicator, HealthIndicatorResult, HealthCheckError } from "@nestjs/terminus"; -import { JsonRpcProviderBase } from "../rpcProvider"; +import { ConfigService } from "@nestjs/config"; +import { Logger } from "@nestjs/common"; +import { HttpService } from "@nestjs/axios"; +import { catchError, firstValueFrom } from "rxjs"; +import { AxiosError } from "axios"; @Injectable() export class JsonRpcHealthIndicator extends HealthIndicator { - constructor(private readonly provider: JsonRpcProviderBase) { + private readonly rpcUrl: string; + private readonly healthCheckTimeoutMs: number; + private readonly logger: Logger; + + constructor(configService: ConfigService, private readonly httpService: HttpService) { super(); + this.logger = new Logger(JsonRpcHealthIndicator.name); + this.rpcUrl = configService.get("blockchain.rpcUrl"); + this.healthCheckTimeoutMs = configService.get("healthChecks.rpcHealthCheckTimeoutMs"); } async isHealthy(key: string): Promise { - const rpcProviderState = this.provider.getState(); - const isHealthy = rpcProviderState === "open"; - const result = this.getStatus(key, isHealthy, { rpcProviderState }); + let isHealthy = true; + try { + // Check RPC health with a pure HTTP request to remove SDK out of the picture + // and avoid any SDK-specific issues. + // Use eth_chainId call as it is the lightest one and return a static value from the memory. + await firstValueFrom( + this.httpService + .post( + this.rpcUrl, + { + id: 1, + jsonrpc: "2.0", + method: "eth_chainId", + params: [], + }, + { timeout: this.healthCheckTimeoutMs } + ) + .pipe( + catchError((error: AxiosError) => { + this.logger.error({ + message: `Failed to ping RPC`, + stack: error.stack, + status: error.response?.status, + response: error.response?.data, + }); + throw error; + }) + ) + ); + } catch { + isHealthy = false; + } + + const result = this.getStatus(key, isHealthy, { status: isHealthy ? "up" : "down" }); if (isHealthy) { return result; } - throw new HealthCheckError("JSON RPC provider is not in open state", result); + throw new HealthCheckError("JSON RPC provider is down or not reachable", result); } } diff --git a/packages/worker/.env.example b/packages/worker/.env.example index d9773b9d8b..9a551c5b21 100644 --- a/packages/worker/.env.example +++ b/packages/worker/.env.example @@ -30,6 +30,9 @@ RPC_BATCH_STALL_TIME_MS=0 COLLECT_DB_CONNECTION_POOL_METRICS_INTERVAL=10000 COLLECT_BLOCKS_TO_PROCESS_METRIC_INTERVAL=10000 +RPC_HEALTH_CHECK_TIMEOUT_MS=20000 +DB_HEALTH_CHECK_TIMEOUT_MS=20000 + DISABLE_MISSING_BLOCKS_METRIC=false CHECK_MISSING_BLOCKS_METRIC_INTERVAL=86400000 diff --git a/packages/worker/src/config.spec.ts b/packages/worker/src/config.spec.ts index e6d561388e..7c678217c7 100644 --- a/packages/worker/src/config.spec.ts +++ b/packages/worker/src/config.spec.ts @@ -61,6 +61,10 @@ describe("config", () => { interval: 86_400_000, }, }, + healthChecks: { + rpcHealthCheckTimeoutMs: 20_000, + dbHealthCheckTimeoutMs: 20_000, + }, }; }); @@ -123,6 +127,10 @@ describe("config", () => { interval: 86_400_000, }, }, + healthChecks: { + rpcHealthCheckTimeoutMs: 20_000, + dbHealthCheckTimeoutMs: 20_000, + }, }); }); diff --git a/packages/worker/src/config.ts b/packages/worker/src/config.ts index ba43f72596..b6e83aefd9 100644 --- a/packages/worker/src/config.ts +++ b/packages/worker/src/config.ts @@ -33,6 +33,8 @@ export default () => { COINGECKO_API_KEY, DISABLE_MISSING_BLOCKS_METRIC, CHECK_MISSING_BLOCKS_METRIC_INTERVAL, + RPC_HEALTH_CHECK_TIMEOUT_MS, + DB_HEALTH_CHECK_TIMEOUT_MS, } = process.env; return { @@ -97,5 +99,9 @@ export default () => { interval: parseInt(CHECK_MISSING_BLOCKS_METRIC_INTERVAL, 10) || 86_400_000, // 1 day }, }, + healthChecks: { + rpcHealthCheckTimeoutMs: parseInt(RPC_HEALTH_CHECK_TIMEOUT_MS, 10) || 20_000, + dbHealthCheckTimeoutMs: parseInt(DB_HEALTH_CHECK_TIMEOUT_MS, 10) || 20_000, + }, }; }; diff --git a/packages/worker/src/health/health.controller.spec.ts b/packages/worker/src/health/health.controller.spec.ts index 6dc59e2200..a7b47551ea 100644 --- a/packages/worker/src/health/health.controller.spec.ts +++ b/packages/worker/src/health/health.controller.spec.ts @@ -4,12 +4,14 @@ import { HealthCheckService, TypeOrmHealthIndicator, HealthCheckResult } from "@ import { mock } from "jest-mock-extended"; import { JsonRpcHealthIndicator } from "./jsonRpcProvider.health"; import { HealthController } from "./health.controller"; +import { ConfigService } from "@nestjs/config"; describe("HealthController", () => { let healthCheckServiceMock: HealthCheckService; let dbHealthCheckerMock: TypeOrmHealthIndicator; let jsonRpcHealthIndicatorMock: JsonRpcHealthIndicator; let healthController: HealthController; + let configServiceMock: ConfigService; beforeEach(async () => { healthCheckServiceMock = mock({ @@ -20,6 +22,12 @@ describe("HealthController", () => { }), }); + configServiceMock = mock({ + get: jest.fn().mockImplementation((key: string) => { + if (key === "healthChecks.dbHealthCheckTimeoutMs") return 5000; + return null; + }), + }); dbHealthCheckerMock = mock(); jsonRpcHealthIndicatorMock = mock(); @@ -38,6 +46,10 @@ describe("HealthController", () => { provide: JsonRpcHealthIndicator, useValue: jsonRpcHealthIndicatorMock, }, + { + provide: ConfigService, + useValue: configServiceMock, + }, ], }).compile(); @@ -50,7 +62,7 @@ describe("HealthController", () => { it("checks health of the DB", async () => { await healthController.check(); expect(dbHealthCheckerMock.pingCheck).toHaveBeenCalledTimes(1); - expect(dbHealthCheckerMock.pingCheck).toHaveBeenCalledWith("database"); + expect(dbHealthCheckerMock.pingCheck).toHaveBeenCalledWith("database", { timeout: 5000 }); }); it("checks health of the JSON RPC provider", async () => { diff --git a/packages/worker/src/health/health.controller.ts b/packages/worker/src/health/health.controller.ts index ae5678e812..15c34bb98d 100644 --- a/packages/worker/src/health/health.controller.ts +++ b/packages/worker/src/health/health.controller.ts @@ -1,17 +1,21 @@ import { Logger, Controller, Get } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; import { HealthCheckService, TypeOrmHealthIndicator, HealthCheck, HealthCheckResult } from "@nestjs/terminus"; import { JsonRpcHealthIndicator } from "./jsonRpcProvider.health"; @Controller(["health", "ready"]) export class HealthController { private readonly logger: Logger; + private readonly dbHealthCheckTimeoutMs: number; constructor( private readonly healthCheckService: HealthCheckService, private readonly dbHealthChecker: TypeOrmHealthIndicator, - private readonly jsonRpcHealthIndicator: JsonRpcHealthIndicator + private readonly jsonRpcHealthIndicator: JsonRpcHealthIndicator, + configService: ConfigService ) { this.logger = new Logger(HealthController.name); + this.dbHealthCheckTimeoutMs = configService.get("healthChecks.dbHealthCheckTimeoutMs"); } @Get() @@ -19,7 +23,7 @@ export class HealthController { public async check(): Promise { try { return await this.healthCheckService.check([ - () => this.dbHealthChecker.pingCheck("database"), + () => this.dbHealthChecker.pingCheck("database", { timeout: this.dbHealthCheckTimeoutMs }), () => this.jsonRpcHealthIndicator.isHealthy("jsonRpcProvider"), ]); } catch (error) { diff --git a/packages/worker/src/health/health.module.ts b/packages/worker/src/health/health.module.ts index dae128825d..37f46aac46 100644 --- a/packages/worker/src/health/health.module.ts +++ b/packages/worker/src/health/health.module.ts @@ -2,10 +2,11 @@ import { Module } from "@nestjs/common"; import { TerminusModule } from "@nestjs/terminus"; import { HealthController } from "./health.controller"; import { JsonRpcHealthIndicator } from "./jsonRpcProvider.health"; +import { HttpModule } from "@nestjs/axios"; @Module({ controllers: [HealthController], - imports: [TerminusModule], + imports: [TerminusModule, HttpModule], providers: [JsonRpcHealthIndicator], }) export class HealthModule {} diff --git a/packages/worker/src/health/jsonRpcProvider.health.spec.ts b/packages/worker/src/health/jsonRpcProvider.health.spec.ts index 2cfaa28919..e495025bbb 100644 --- a/packages/worker/src/health/jsonRpcProvider.health.spec.ts +++ b/packages/worker/src/health/jsonRpcProvider.health.spec.ts @@ -1,17 +1,20 @@ import { Test, TestingModule } from "@nestjs/testing"; +import { Logger } from "@nestjs/common"; import { mock } from "jest-mock-extended"; -import { HealthCheckError } from "@nestjs/terminus"; import { JsonRpcProviderBase } from "../rpcProvider"; import { JsonRpcHealthIndicator } from "./jsonRpcProvider.health"; +import { ConfigService } from "@nestjs/config"; +import { HttpService } from "@nestjs/axios"; +import { of, throwError } from "rxjs"; +import { AxiosError } from "axios"; describe("JsonRpcHealthIndicator", () => { - const healthIndicatorKey = "rpcProvider"; let jsonRpcProviderMock: JsonRpcProviderBase; let jsonRpcHealthIndicator: JsonRpcHealthIndicator; + let httpService: HttpService; + let configService: ConfigService; - beforeEach(async () => { - jsonRpcProviderMock = mock(); - + const getHealthIndicator = async () => { const app: TestingModule = await Test.createTestingModule({ providers: [ JsonRpcHealthIndicator, @@ -19,38 +22,90 @@ describe("JsonRpcHealthIndicator", () => { provide: JsonRpcProviderBase, useValue: jsonRpcProviderMock, }, + { + provide: HttpService, + useValue: httpService, + }, + { + provide: ConfigService, + useValue: configService, + }, ], }).compile(); - jsonRpcHealthIndicator = app.get(JsonRpcHealthIndicator); + app.useLogger(mock()); + return app.get(JsonRpcHealthIndicator); + }; + + beforeEach(async () => { + jsonRpcProviderMock = mock(); + + httpService = mock({ + post: jest.fn(), + }); + + configService = mock({ + get: jest.fn().mockImplementation((key: string) => { + if (key === "blockchain.rpcUrl") return "http://localhost:3050"; + if (key === "healthChecks.rpcHealthCheckTimeoutMs") return 5000; + return null; + }), + }); + + jsonRpcHealthIndicator = await getHealthIndicator(); }); describe("isHealthy", () => { - describe("when rpcProvider is open", () => { - beforeEach(() => { - jest.spyOn(jsonRpcProviderMock, "getState").mockReturnValueOnce("open"); - }); + const rpcRequest = { + id: 1, + jsonrpc: "2.0", + method: "eth_chainId", + params: [], + }; - it("returns OK health indicator result", async () => { - const result = await jsonRpcHealthIndicator.isHealthy(healthIndicatorKey); - expect(result).toEqual({ [healthIndicatorKey]: { rpcProviderState: "open", status: "up" } }); + it("returns healthy status when RPC responds successfully", async () => { + (httpService.post as jest.Mock).mockReturnValueOnce(of({ data: { result: "0x1" } })); + const result = await jsonRpcHealthIndicator.isHealthy("jsonRpcProvider"); + expect(result).toEqual({ + jsonRpcProvider: { + status: "up", + }, }); + expect(httpService.post).toHaveBeenCalledWith("http://localhost:3050", rpcRequest, { timeout: 5000 }); }); - describe("when rpcProvider is closed", () => { - beforeEach(() => { - jest.spyOn(jsonRpcProviderMock, "getState").mockReturnValueOnce("closed"); - }); + it("throws HealthCheckError when RPC request fails", async () => { + const error = new AxiosError(); + error.response = { + status: 503, + data: "Service Unavailable", + } as any; - it("throws HealthCheckError error", async () => { - expect.assertions(2); - try { - await jsonRpcHealthIndicator.isHealthy(healthIndicatorKey); - } catch (error) { - expect(error).toBeInstanceOf(HealthCheckError); - expect(error.message).toBe("JSON RPC provider is not in open state"); - } + (httpService.post as jest.Mock).mockReturnValueOnce(throwError(() => error)); + await expect(jsonRpcHealthIndicator.isHealthy("jsonRpcProvider")).rejects.toThrow(); + expect(httpService.post).toHaveBeenCalledWith("http://localhost:3050", rpcRequest, { timeout: 5000 }); + }); + + it("throws HealthCheckError when RPC request times out", async () => { + const error = new AxiosError(); + error.code = "ECONNABORTED"; + + (httpService.post as jest.Mock).mockReturnValueOnce(throwError(() => error)); + await expect(jsonRpcHealthIndicator.isHealthy("jsonRpcProvider")).rejects.toThrow(); + expect(httpService.post).toHaveBeenCalledWith("http://localhost:3050", rpcRequest, { timeout: 5000 }); + }); + + it("uses configured timeout from config service", async () => { + (configService.get as jest.Mock).mockImplementation((key: string) => { + if (key === "blockchain.rpcUrl") return "http://localhost:3050"; + if (key === "healthChecks.rpcHealthCheckTimeoutMs") return 10000; + return null; }); + jsonRpcHealthIndicator = await getHealthIndicator(); + + (httpService.post as jest.Mock).mockReturnValueOnce(of({ data: { result: "0x1" } })); + await jsonRpcHealthIndicator.isHealthy("jsonRpcProvider"); + expect(httpService.post).toHaveBeenCalledWith("http://localhost:3050", rpcRequest, { timeout: 10000 }); }); }); }); diff --git a/packages/worker/src/health/jsonRpcProvider.health.ts b/packages/worker/src/health/jsonRpcProvider.health.ts index 60406bfb4c..e5434382b2 100644 --- a/packages/worker/src/health/jsonRpcProvider.health.ts +++ b/packages/worker/src/health/jsonRpcProvider.health.ts @@ -1,22 +1,64 @@ import { Injectable } from "@nestjs/common"; import { HealthIndicator, HealthIndicatorResult, HealthCheckError } from "@nestjs/terminus"; -import { JsonRpcProviderBase } from "../rpcProvider"; +import { ConfigService } from "@nestjs/config"; +import { Logger } from "@nestjs/common"; +import { HttpService } from "@nestjs/axios"; +import { catchError, firstValueFrom } from "rxjs"; +import { AxiosError } from "axios"; @Injectable() export class JsonRpcHealthIndicator extends HealthIndicator { - constructor(private readonly provider: JsonRpcProviderBase) { + private readonly rpcUrl: string; + private readonly healthCheckTimeoutMs: number; + private readonly logger: Logger; + + constructor(configService: ConfigService, private readonly httpService: HttpService) { super(); + this.logger = new Logger(JsonRpcHealthIndicator.name); + this.rpcUrl = configService.get("blockchain.rpcUrl"); + this.healthCheckTimeoutMs = configService.get("healthChecks.rpcHealthCheckTimeoutMs"); } async isHealthy(key: string): Promise { - const rpcProviderState = this.provider.getState(); - const isHealthy = rpcProviderState === "open"; - const result = this.getStatus(key, isHealthy, { rpcProviderState }); + let isHealthy = true; + try { + // Check RPC health with a pure HTTP request to remove SDK out of the picture + // and avoid any SDK-specific issues. + // Use eth_chainId call as it is the lightest one and return a static value from the memory. + await firstValueFrom( + this.httpService + .post( + this.rpcUrl, + { + id: 1, + jsonrpc: "2.0", + method: "eth_chainId", + params: [], + }, + { timeout: this.healthCheckTimeoutMs } + ) + .pipe( + catchError((error: AxiosError) => { + this.logger.error({ + message: `Failed to ping RPC`, + stack: error.stack, + status: error.response?.status, + response: error.response?.data, + }); + throw error; + }) + ) + ); + } catch { + isHealthy = false; + } + + const result = this.getStatus(key, isHealthy, { status: isHealthy ? "up" : "down" }); if (isHealthy) { return result; } - throw new HealthCheckError("JSON RPC provider is not in open state", result); + throw new HealthCheckError("JSON RPC provider is down or not reachable", result); } }