Example client block cache

example/package.json

@@ -11,6 +11,8 @@
   "license": "ISC",
   "devDependencies": {
     "dotenv": "^16.3.1",
+    "leveldown": "^6.1.1",
+    "levelup": "^5.1.1",
     "ts-node": "^10.9.1",
     "typescript": "^5.1.6"
   }

example/src/utils/AccountProcessor.ts

@@ -0,0 +1,68 @@
+import { generateKeyFromPrivateKey, Key } from "@ironfish/rust-nodejs";
+import { NoteEncrypted } from "@ironfish/sdk/build/src/primitives/noteEncrypted";
+import { LightBlock } from "../../../src/models/lightstreamer";
+
+interface AccountData {
+  key: Key;
+  assetBalances: Map<string, bigint>;
+}
+
+/**
+ * Mapping of private key to account data including keys and asset balances
+ */
+type TAccounts = Map<
+  string,
+  {
+    key: Key;
+    assetBalances: Map<string, bigint>;
+  }
+>;
+
+export class AccountsManager {
+  private accounts: TAccounts = new Map();
+
+  public addAccount(privateKey: string) {
+    this.accounts.set(...this._makeAccountData(privateKey));
+  }
+
+  public getPublicAddresses() {
+    return Array.from(this.accounts.keys());
+  }
+
+  private _makeAccountData(privateKey: string): [string, AccountData] {
+    const key = generateKeyFromPrivateKey(privateKey);
+    return [
+      key.publicAddress,
+      {
+        key,
+        assetBalances: new Map(),
+      },
+    ];
+  }
+
+  public processBlockForTransactions(block: LightBlock) {
+    block.transactions.forEach((tx) => {
+      tx.outputs.forEach((output) => {
+        this._processNote(new NoteEncrypted(output.note));
+      });
+
+      // @todo: Process spends
+    });
+  }
+
+  private _processNote(note: NoteEncrypted) {
+    for (const publicKey of this.accounts.keys()) {
+      const result = note.decryptNoteForOwner(publicKey);
+      if (!result) return;
+
+      const account = this.accounts.get(publicKey);
+      if (!account) return;
+
+      const assetId = result.assetId().toString("hex");
+      const amount = result.value();
+
+      const currentBalance = account.assetBalances.get(assetId) ?? BigInt(0);
+      account.assetBalances.set(assetId, currentBalance + amount);
+    }
+  }
+}

example/src/utils/BlockCache.ts

@@ -0,0 +1,54 @@
+import levelup, { LevelUp } from "levelup";
+import leveldown from "leveldown";
+import path from "path";
+import { LightBlock } from "../../../src/models/lightstreamer";
+
+const KNOWN_KEYS = {
+  HEAD_SEQUENCE: "__HEAD_SEQUENCE__",
+};
+
+// Storing keys as zero-padded numbers to avoid lexicographic ordering.
+// At one minute block times, this gives us ~1,900 years of blocks.
+const KEY_LENGTH = 9;
+
+export class BlockCache {
+  private db: LevelUp;
+
+  constructor() {
+    this.db = levelup(
+      leveldown(path.join(__dirname, "..", "client-block-cache")),
+    );
+  }
+
+  public async getHeadSequence() {
+    try {
+      const headSequence = await this.db.get(KNOWN_KEYS.HEAD_SEQUENCE);
+      const asNumber = Number(headSequence);
+      if (isNaN(asNumber)) {
+        throw new Error("Head sequence is not a number");
+      }
+      return asNumber;
+    } catch (_err) {
+      return 0;
+    }
+  }
+
+  public cacheBlock(block: LightBlock) {
+    const sequence = block.sequence;
+    console.log(`Caching block ${sequence}`);
+
+    this.db
+      .batch()
+      .put(this.encodeKey(sequence), block)
+      .put(KNOWN_KEYS.HEAD_SEQUENCE, sequence)
+      .write();
+  }
+
+  public encodeKey(num: number) {
+    return num.toString().padStart(KEY_LENGTH, "0");
+  }
+
+  public decodeKey(key: string) {
+    return Number(key);
+  }
+}

example/src/utils/BlockProcessor.ts

@@ -1,49 +1,38 @@
+import { ServiceError } from "@grpc/grpc-js";
 import { NoteEncrypted } from "@ironfish/sdk/build/src/primitives/noteEncrypted";
 import {
   BlockID,
   Empty,
   LightBlock,
   LightStreamerClient,
 } from "../../../src/models/lightstreamer";
-import { ServiceError } from "@grpc/grpc-js";
+import { BlockCache } from "./BlockCache";
 
 function addToMerkleTree(note: NoteEncrypted) {
   return note;
 }
 
 const POLL_INTERVAL = 30 * 1000;
 
-/**
- * @todo:
- * Reorgs =>
- *   To determine if re-org happened, when querying new blocks, check that each block's prev block hash
- *   matches the previous block's block hash. If it does not, walk back until you find a block that matches.
- * Store transactions =>
- *   Add simple DB to store transactions so we don't have to start querying from block 1 when restarting or
- *   importing a new account.
- * Account balances =>
- *   Add example of processing notes to deterine account balances.
- * Add error handling to server if unable to connect to node.
- */
-
 export class BlockProcessor {
   private client: LightStreamerClient;
   private pollInterval?: NodeJS.Timer;
   private handleStop?: () => void;
   private isProcessingBlocks: boolean = false;
-  private lastProcessedBlock: number = 0;
+  private blockCache = new BlockCache();
 
   constructor(client: LightStreamerClient) {
     this.client = client;
   }
 
-  public start() {
+  public async start() {
     if (this.pollInterval !== undefined) {
       console.warn("Process already running");
       return;
     }
 
     this._pollForNewBlocks();
+
     this.pollInterval = setInterval(
       this._pollForNewBlocks.bind(this),
       POLL_INTERVAL,
@@ -78,24 +67,28 @@ export class BlockProcessor {
       throw new Error("Head sequence is undefined");
     }
 
-    if (headSequence === this.lastProcessedBlock) {
+    const cachedHeadSequence = await this.blockCache.getHeadSequence();
+
+    if (headSequence === cachedHeadSequence) {
       return;
     }
 
-    await this._processBlockRange(this.lastProcessedBlock + 1, headSequence);
+    await this._processBlockRange(cachedHeadSequence + 1, headSequence);
 
     this.isProcessingBlocks = false;
   }
 
   private _getLatestBlock() {
     return new Promise<[ServiceError | null, BlockID]>((res) => {
-      this.client.getLatestBlock(Empty, (error, result) =>
-        res([error, result]),
-      );
+      this.client.getLatestBlock(Empty, (error, result) => {
+        res([error, result]);
+      });
     });
   }
 
   private async _processBlockRange(startSequence: number, endSequence: number) {
+    console.log(`Processing blocks from ${startSequence} to ${endSequence}`);
+
     const stream = this.client.getBlockRange({
       start: {
         sequence: startSequence,
@@ -121,13 +114,13 @@ export class BlockProcessor {
   }
 
   private _processBlock(block: LightBlock) {
+    this.blockCache.cacheBlock(block);
+
     for (const transaction of block.transactions) {
       for (const output of transaction.outputs) {
         const note = new NoteEncrypted(output.note);
         addToMerkleTree(note);
       }
     }
-
-    this.lastProcessedBlock = block.sequence;
   }
 }