feat: health probe

src/domain/chainContext.ts

@@ -24,6 +24,30 @@ const WATCHDOG_FREQUENCY = 5 * 1000; // 5 seconds
 
 const MULTICALL3 = "0xcA11bde05977b3631167028862bE2a173976CA11";
 
+enum ChainSync {
+  SYNCED = "SYNCED",
+  SYNCING = "SYNCING",
+}
+
+type Chains = { [chainId: number]: ChainContext };
+
+export interface ChainStatus {
+  sync: ChainSync;
+  chainId: SupportedChainId;
+  lastProcessedBlock: number;
+}
+
+export interface ChainHealth extends ChainStatus {
+  isHealthy: boolean;
+}
+
+export interface ChainWatcherHealth {
+  overallHealth: boolean;
+  chains: {
+    [chainId: number]: ChainHealth;
+  };
+}
+
 /**
  * The chain context handles watching a single chain for new conditional orders
  * and executing them.
@@ -32,7 +56,8 @@ export class ChainContext {
   readonly deploymentBlock: number;
   readonly pageSize: number;
   readonly dryRun: boolean;
-  private inSync = false;
+  private sync: ChainSync = ChainSync.SYNCING;
+  static chains: Chains = {};
 
   provider: ethers.providers.Provider;
   chainId: SupportedChainId;
@@ -82,7 +107,11 @@ export class ChainContext {
       deploymentBlock
     );
 
-    return new ChainContext(options, provider, chainId, registry);
+    // Save the context to the static map to be used by the API
+    const context = new ChainContext(options, provider, chainId, registry);
+    ChainContext.chains[chainId] = context;
+
+    return context;
   }
 
   /**
@@ -196,14 +225,13 @@ export class ChainContext {
 
       // If we are in sync, let it be known
       if (currentBlockNumber === this.registry.lastProcessedBlock) {
-        this.inSync = true;
+        this.sync = ChainSync.SYNCED;
       } else {
         // Otherwise, we need to keep processing blocks
-        this.inSync = false;
         fromBlock = this.registry.lastProcessedBlock + 1;
         plan = {};
       }
-    } while (!this.inSync);
+    } while (this.sync === ChainSync.SYNCING);
 
     log.info(
       `💚 ${
@@ -263,6 +291,7 @@ export class ChainContext {
 
           // Block height metric
           blockHeight.labels(chainId.toString()).set(Number(blockNumber));
+          this.registry.lastProcessedBlock = Number(blockNumber);
         } catch {
           log.error(`Error processing block ${blockNumber}`);
         }
@@ -295,6 +324,26 @@ export class ChainContext {
       }
     }
   }
+
+  get status(): ChainStatus {
+    const { sync, chainId } = this;
+    return {
+      sync,
+      chainId,
+      lastProcessedBlock: this.registry.lastProcessedBlock ?? 0,
+    };
+  }
+
+  get health(): ChainHealth {
+    return {
+      ...this.status,
+      isHealthy: this.isHealthy(),
+    };
+  }
+
+  private isHealthy(): boolean {
+    return this.sync === ChainSync.SYNCED;
+  }
 }
 
 /**

src/utils/api.ts

@@ -6,6 +6,7 @@ import { getLogger } from "./logging";
 import { DBService } from "./db";
 import { Registry } from "../types";
 import { version, name, description } from "../../package.json";
+import { ChainContext, ChainHealth } from "../domain";
 
 export class ApiService {
   protected port: number;
@@ -33,6 +34,22 @@ export class ApiService {
     this.app.get("/", (_req: Request, res: Response) => {
       res.send("🐮 Moooo!");
     });
+    this.app.get("/health", async (_req: Request, res: Response) => {
+      // Using an iterator, process all the chain contexts, storing the health
+      // in a map, and if any of the contexts are unhealthy, return false
+      const contexts = Object.values(ChainContext.chains) as ChainContext[];
+      const healths = contexts.map((context) => context.health);
+      const healthStatusDict = healths.reduce((acc, health) => {
+        acc[health.chainId] = health;
+        return acc;
+      }, {} as { [chainId: string]: ChainHealth });
+
+      const overallHealth = healths.every((health) => health.isHealthy);
+
+      res
+        .status(overallHealth ? 200 : 500)
+        .send({ overallHealth, ...healthStatusDict });
+    });
     this.app.use("/api", router);
   }