diff --git a/src/cache/index.test.slow.ts b/src/cache/index.test.slow.ts index 99c4184..20443e1 100644 --- a/src/cache/index.test.slow.ts +++ b/src/cache/index.test.slow.ts @@ -14,7 +14,7 @@ describe("LightBlockCache creating cache", () => { const cacheBlocks = lightBlockCache.cacheBlocks(); await Promise.race([cacheBlocks, delay()]); const head = await lightBlockCache.getHead(); - const block = await lightBlockCache.getBlockByHash(head!.toString("hex")); + const block = await lightBlockCache.getLightBlock(head!); expect(block).toHaveProperty("protoVersion"); expect(block).toHaveProperty("sequence"); expect(block).toHaveProperty("hash"); diff --git a/src/cache/index.test.ts b/src/cache/index.test.ts index 9e63434..70cfc81 100644 --- a/src/cache/index.test.ts +++ b/src/cache/index.test.ts @@ -1,10 +1,7 @@ -import { LightBlock } from "../models/lightstreamer"; import { blockFixture } from "../../test/fixtures"; import { lightBlockCache } from "."; describe("LightBlockCache", () => { - const fakeHash = "hash1"; - beforeAll(async () => { await lightBlockCache.open(); }); @@ -14,24 +11,25 @@ describe("LightBlockCache", () => { }); it("storing and retrieving block is successful", async () => { - const encoded = LightBlock.encode(blockFixture).finish(); - await lightBlockCache.put(fakeHash, encoded); - await lightBlockCache.put(blockFixture.sequence.toString(), fakeHash); + await lightBlockCache.putLightBlock(blockFixture); - const hashBlock = await lightBlockCache.getBlockByHash(fakeHash); + const hashBlock = await lightBlockCache.getLightBlock(blockFixture.hash); expect(hashBlock).toEqual(blockFixture); - const sequenceBlock = await lightBlockCache.getBlockBySequence( + const sequenceBlock = await lightBlockCache.getLightBlockBySequence( blockFixture.sequence, ); expect(sequenceBlock).toEqual(blockFixture); }); it("storing and retrieving hash is successful", async () => { - await lightBlockCache.put("head", Buffer.from("deedbeef", "hex")); + await lightBlockCache.putHead(Buffer.from("deadbeef", "hex"), 1000); + + const head = await lightBlockCache.getHead(); + expect(head?.toString("hex")).toEqual("deadbeef"); - const block = await lightBlockCache.getHead(); - expect(block!.toString("hex")).toEqual("deedbeef"); + const sequence = await lightBlockCache.getHeadSequence(); + expect(sequence).toEqual(1000); }); it("finality sequence is always behind head sequence by specified amount", async () => { diff --git a/src/cache/index.ts b/src/cache/index.ts index b57230f..e6fb975 100644 --- a/src/cache/index.ts +++ b/src/cache/index.ts @@ -83,31 +83,29 @@ export class LightBlockCache { } else if (content.type === "disconnected") { logger.warn(`Removing block ${content.block.sequence}...`); const block = lightBlock(content); - await this.db.put("head", block.previousBlockHash); - await this.db.put("headSequence", (block.sequence - 1).toString()); - await this.db.del(block.sequence); - await this.db.del(block.hash); + await this.putHead(block.previousBlockHash, block.sequence - 1); + await this.del(block.sequence.toString()); + await this.del(block.hash.toString("hex")); } } } } private async rollbackHead(): Promise { - let head = (await this.getHeadSequence()) - 1; - if (!head) { + let headSequence = (await this.getHeadSequence()) - 1; + if (!headSequence) { logger.error("Head sequence is not set. Cannot rollback."); return; } let block = null; while (!block) { - block = await this.getBlockBySequence(head); + block = await this.getLightBlockBySequence(headSequence); if (!block) { - head -= 1; + headSequence -= 1; } } - await this.db.put("headSequence", head.toString()); - await this.db.put("head", block.hash); - logger.info(`Rolled back head to block sequence ${head}`); + await this.putHead(block.hash, headSequence); + logger.info(`Rolled back head to block sequence ${headSequence}`); } async cacheBlock(block: LightBlock): Promise { @@ -118,14 +116,12 @@ export class LightBlockCache { ); } const hash = block.hash; - await this.db.put(hash, LightBlock.encode(block).finish()); - await this.db.put(block.sequence.toString(), hash); + await this.putLightBlock(block); const finalizedSequence = await this.getFinalizedBlockSequence(); if (block.sequence - this.finalityBlockCount > finalizedSequence) { this.putFinalizedBlockSequence(block.sequence - this.finalityBlockCount); } - await this.db.put("head", hash); - await this.db.put("headSequence", block.sequence.toString()); + await this.putHead(hash, block.sequence); } async getFinalizedBlockSequence(): Promise { @@ -135,23 +131,32 @@ export class LightBlockCache { : this.finalityBlockCount + 1; } - async getHead(): Promise { - const head = await this.get("head"); - return head ? Buffer.from(head) : null; + async putFinalizedBlockSequence(sequence: number): Promise { + await this.put("finalizedBlockSequence", Buffer.from(sequence.toString())); } - async putFinalizedBlockSequence(sequence: number): Promise { - await this.db.put("finalizedBlockSequence", sequence.toString()); + async putHead(hash: Buffer, sequence: number): Promise { + await this.put("head", hash); + await this.put("headSequence", Buffer.from(sequence.toString())); } - async getBlockByHash(hash: string): Promise { - const block = await this.get(hash); - return block ? LightBlock.decode(block) : null; + async getLightBlock(hash: Buffer): Promise { + try { + const data = await this.get(hash.toString("hex")); + if (!data) return null; + return LightBlock.decode(data); + } catch (e) { + return null; + } } - async getBlockBySequence(sequence: number): Promise { + async getLightBlockBySequence(sequence: number): Promise { const hash = await this.get(sequence.toString()); - return hash ? await this.getBlockByHash(hash.toString()) : null; + return hash ? await this.getLightBlock(hash) : null; + } + + async getHead(): Promise { + return this.get("head"); } async getHeadSequence(): Promise { @@ -160,7 +165,7 @@ export class LightBlockCache { return Number(head.toString()); } - async get(key: string): Promise { + async get(key: string): Promise { try { const data = await this.db.get(key); return data; @@ -169,10 +174,17 @@ export class LightBlockCache { } } - async put(key: string, value: Uint8Array | string): Promise { + private async put(key: string, value: Buffer): Promise { await this.db.put(key, value); } + async putLightBlock(block: LightBlock): Promise { + const key = block.hash.toString("hex"); + const value = LightBlock.encode(block).finish(); + await this.put(block.sequence.toString(), block.hash); + await this.put(key, Buffer.from(value)); + } + async del(key: string): Promise { await this.db.del(key); } diff --git a/src/controllers/block.ts b/src/controllers/block.ts index 9c0bb68..1444f42 100644 --- a/src/controllers/block.ts +++ b/src/controllers/block.ts @@ -75,9 +75,9 @@ export class BlockController { let block = null; if (hash) { - block = await lightBlockCache.getBlockByHash(String(hash)); + block = await lightBlockCache.getLightBlock(Buffer.from(hash, "hex")); } else if (sequence) { - block = await lightBlockCache.getBlockBySequence(Number(sequence)); + block = await lightBlockCache.getLightBlockBySequence(Number(sequence)); } if (block) { @@ -127,7 +127,7 @@ export class BlockController { // Placeholder logic: Fetch blocks from cache or database const blocks: LightBlock[] = []; for (let i = start; i <= end; i++) { - const block = await lightBlockCache.getBlockBySequence(i); + const block = await lightBlockCache.getLightBlockBySequence(i); if (block) { blocks.push(block); } diff --git a/src/upload/index.test.ts b/src/upload/index.test.ts index 5829b8c..a2e302f 100644 --- a/src/upload/index.test.ts +++ b/src/upload/index.test.ts @@ -22,7 +22,7 @@ describe("LightBlockUpload", () => { it("should gzip blocks/manifest, last block should match", async () => { const tempFile = path.join(os.tmpdir(), "test"); - const firstBlock = (await lightBlockCache.getBlockBySequence( + const firstBlock = (await lightBlockCache.getLightBlockBySequence( 1, )) as LightBlock; const triggerUploadTime = diff --git a/src/upload/index.ts b/src/upload/index.ts index 3c89935..e483893 100644 --- a/src/upload/index.ts +++ b/src/upload/index.ts @@ -166,7 +166,7 @@ export class LightBlockUpload { // eslint-disable-next-line no-constant-condition while (true) { - const block = await this.cache.getBlockBySequence(currentSequence); + const block = await this.cache.getLightBlockBySequence(currentSequence); const finalizedSequence = await this.cache.getFinalizedBlockSequence(); if (!block || block.sequence > finalizedSequence) { const currentTimestamp = Date.now(); diff --git a/test/dbfixture/cache/leveldb/000003.log b/test/dbfixture/cache/leveldb/000003.log deleted file mode 100644 index cd87912..0000000 Binary files a/test/dbfixture/cache/leveldb/000003.log and /dev/null differ diff --git a/test/dbfixture/cache/leveldb/000007.ldb b/test/dbfixture/cache/leveldb/000007.ldb new file mode 100644 index 0000000..6862047 Binary files /dev/null and b/test/dbfixture/cache/leveldb/000007.ldb differ diff --git a/test/dbfixture/cache/leveldb/000008.log b/test/dbfixture/cache/leveldb/000008.log new file mode 100644 index 0000000..e69de29 diff --git a/test/dbfixture/cache/leveldb/CURRENT b/test/dbfixture/cache/leveldb/CURRENT index 1a84852..f7753e2 100644 --- a/test/dbfixture/cache/leveldb/CURRENT +++ b/test/dbfixture/cache/leveldb/CURRENT @@ -1 +1 @@ -MANIFEST-000002 +MANIFEST-000006 diff --git a/test/dbfixture/cache/leveldb/LOG b/test/dbfixture/cache/leveldb/LOG index 0329f29..234dfd5 100644 --- a/test/dbfixture/cache/leveldb/LOG +++ b/test/dbfixture/cache/leveldb/LOG @@ -1 +1,5 @@ -2024/03/29-16:12:39.733354 170903000 Delete type=3 #1 +2024/04/08-20:26:12.744956 17269f000 Recovering log #5 +2024/04/08-20:26:12.745468 17269f000 Level-0 table #7: started +2024/04/08-20:26:12.745659 17269f000 Level-0 table #7: 6076 bytes OK +2024/04/08-20:26:12.746007 17269f000 Delete type=0 #5 +2024/04/08-20:26:12.746084 17269f000 Delete type=3 #4 diff --git a/test/dbfixture/cache/leveldb/LOG.old b/test/dbfixture/cache/leveldb/LOG.old new file mode 100644 index 0000000..4e06b12 --- /dev/null +++ b/test/dbfixture/cache/leveldb/LOG.old @@ -0,0 +1,3 @@ +2024/04/08-20:25:05.384949 178807000 Recovering log #3 +2024/04/08-20:25:05.385875 178807000 Delete type=0 #3 +2024/04/08-20:25:05.385905 178807000 Delete type=3 #2 diff --git a/test/dbfixture/cache/leveldb/MANIFEST-000002 b/test/dbfixture/cache/leveldb/MANIFEST-000002 deleted file mode 100644 index bbbc585..0000000 Binary files a/test/dbfixture/cache/leveldb/MANIFEST-000002 and /dev/null differ diff --git a/test/dbfixture/cache/leveldb/MANIFEST-000006 b/test/dbfixture/cache/leveldb/MANIFEST-000006 new file mode 100644 index 0000000..2736b68 Binary files /dev/null and b/test/dbfixture/cache/leveldb/MANIFEST-000006 differ