Merge main, fix conflicts

example/src/Client/utils/BlockProcessor.ts

@@ -1,16 +1,13 @@
 import { ServiceError } from "@grpc/grpc-js";
 import { NoteEncrypted } from "@ironfish/sdk/build/src/primitives/noteEncrypted";
+import { BlockCache } from "./BlockCache";
 import {
   BlockID,
   Empty,
   LightBlock,
   LightStreamerClient,
 } from "../../../../src/models/lightstreamer";
-import { BlockCache } from "./BlockCache";
-
-function addToMerkleTree(note: NoteEncrypted) {
-  return note;
-}
+import { addNotesToMerkleTree } from "./merkle";
 
 const POLL_INTERVAL = 30 * 1000;
 
@@ -82,8 +79,7 @@ export class BlockProcessor {
   }
 
   private async _processBlockRange(startSequence: number, endSequence: number) {
-    console.log(`Processing blocks from ${startSequence} to ${endSequence}`);
-
+    let blocksProcessed = startSequence;
     const stream = this.client.getBlockRange({
       start: {
         sequence: startSequence,
@@ -97,6 +93,10 @@ export class BlockProcessor {
       await new Promise((res) => {
         stream.on("data", (block: LightBlock) => {
           this._processBlock(block);
+          blocksProcessed++;
+          if (blocksProcessed % 100 === 0) {
+            console.log(`Processed ${blocksProcessed}/${endSequence} blocks`);
+          }
         });
 
         stream.on("end", () => {
@@ -108,14 +108,18 @@ export class BlockProcessor {
     }
   }
 
-  private _processBlock(block: LightBlock) {
+  private async _processBlock(block: LightBlock) {
     this.blockCache.cacheBlock(block);
 
+    const notes: NoteEncrypted[] = [];
+
     for (const transaction of block.transactions) {
       for (const output of transaction.outputs) {
         const note = new NoteEncrypted(output.note);
-        addToMerkleTree(note);
+        notes.push(note);
       }
     }
+
+    await addNotesToMerkleTree(notes);
   }
 }

example/src/Client/utils/merkle.ts

@@ -0,0 +1,25 @@
+import { NoteHasher } from "@ironfish/sdk/build/src/merkletree/hasher";
+import { MerkleTree } from "@ironfish/sdk/build/src/merkletree/merkletree";
+import { BUFFER_ENCODING } from "@ironfish/sdk/build/src/storage";
+import { createDB } from "@ironfish/sdk/build/src/storage/utils";
+import { LeafEncoding } from "@ironfish/sdk/build/src/merkletree/database/leaves";
+import { NodeEncoding } from "@ironfish/sdk/build/src/merkletree/database/nodes";
+import { NoteEncrypted } from "@ironfish/sdk/build/src/primitives/noteEncrypted";
+
+const db = createDB({ location: "./testdb" });
+db.open();
+
+const notesTree = new MerkleTree({
+  hasher: new NoteHasher(),
+  leafIndexKeyEncoding: BUFFER_ENCODING,
+  leafEncoding: new LeafEncoding(),
+  nodeEncoding: new NodeEncoding(),
+  db,
+  name: "n",
+  depth: 32,
+  defaultValue: Buffer.alloc(32),
+});
+
+export const addNotesToMerkleTree = async (notes: NoteEncrypted[]) => {
+  return notesTree.addBatch(notes);
+};

example/testdb/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000002

example/testdb/LOG

@@ -0,0 +1,4 @@
+2023/08/30-10:57:37.973637 1710bf000 Delete type=3 #1
+2023/08/30-10:57:47.827098 1721e7000 Level-0 table #5: started
+2023/08/30-10:57:47.847238 1721e7000 Level-0 table #5: 2049651 bytes OK
+2023/08/30-10:57:47.847443 1721e7000 Delete type=0 #3

protos/lightstreamer.proto

@@ -22,6 +22,8 @@ message LightBlock {
     bytes previousBlockHash = 4;         // the ID (hash) of this block's predecessor
     uint32 timestamp = 5;            // Unix epoch time when the block was mined
     repeated LightTransaction transactions = 6; // zero or more compact transactions from this block
+    uint64 noteSize = 7; // the size of the notes tree after adding transactions from this block.
+
 }
 
 message LightTransaction {

src/cache/index.ts

@@ -48,6 +48,9 @@ class LightBlockCache {
     });
 
     for await (const content of stream.contentStream()) {
+      if (content.block.sequence % 1000 === 0) {
+        logger.info(`Caching block ${content.block.sequence}`);
+      }
       if (content.type === "connected") {
         if (content.block.sequence % 1000 === 0) {
           logger.info(

src/utils/lightBlock.ts

@@ -13,6 +13,9 @@ import {
 export function lightBlock(
   response: FollowChainStreamResponse | GetBlockResponse,
 ): LightBlock {
+  if (!response.block.noteSize) {
+    throw new Error("Block is missing noteSize");
+  }
   const lightTransactions: LightTransaction[] = [];
   const previousBlockHash =
     "previous" in response.block
@@ -50,5 +53,6 @@ export function lightBlock(
     previousBlockHash: Buffer.from(previousBlockHash, "hex"),
     timestamp: response.block.timestamp,
     transactions: lightTransactions,
+    noteSize: response.block.noteSize,
   };
 }