upload lightblocks

.gitignore

@@ -9,4 +9,6 @@ node_modules/
 *testdb*
 *yarn-error.log
 test/*
-blocks.gz
+blocks
+blocks.gz
+blocks.manifest

protos/lightstreamer.proto

@@ -5,7 +5,7 @@ message Empty {}
 
 message BlockID {
   optional uint64 sequence = 1;
-  optional string hash = 2;
+  optional bytes hash = 2;
 }
 
 // BlockRange specifies a series of blocks from start to end inclusive.
@@ -18,8 +18,8 @@ message BlockRange {
 message LightBlock {
 	uint32 protoVersion = 1;    // the version of this wire format, for storage
   uint64 sequence = 2;          // the height of this block
-  string hash = 3;             // the ID (hash) of this block, same as explorer
-  string previousBlockHash = 4;         // the ID (hash) of this block's predecessor
+  bytes hash = 3;             // the ID (hash) of this block, same as explorer
+  bytes previousBlockHash = 4;         // the ID (hash) of this block's predecessor
   uint32 timestamp = 5;            // Unix epoch time when the block was mined
   repeated LightTransaction transactions = 6; // zero or more compact transactions from this block
   uint64 noteSize = 7; // the size of the notes tree after adding transactions from this block.
@@ -28,28 +28,28 @@ message LightBlock {
 
 message LightTransaction {
   uint64 index = 1; // do we need this field?
-  string hash = 2;
+  bytes hash = 2;
 
   repeated LightSpend spends = 4;
   repeated LightOutput outputs = 5;
 }
 
 
 message LightSpend {
-  string nf = 2;
+  bytes nf = 2;
 }
 
 message LightOutput {
-  string note = 1; // NoteEncrypted, serialized
+  bytes note = 1; // NoteEncrypted, serialized
 }
 
 message Transaction {
   // built, encrypted transaction
-  string data = 1;
+  bytes data = 1;
 }
 
 message SendResponse {
-  string hash = 1;
+  bytes hash = 1;
   bool accepted = 2;
 }
 

src/server.test.ts

@@ -9,15 +9,15 @@ import { LightBlock } from "./models/lightstreamer";
 const expectedBlockObject = {
   protoVersion: expect.any(Number),
   sequence: 1,
-  hash: expect.any(String),
-  previousBlockHash: expect.any(String),
+  hash: expect.any(Buffer),
+  previousBlockHash: expect.any(Buffer),
   timestamp: expect.any(Number),
   transactions: expect.arrayContaining([
     expect.objectContaining({
-      hash: expect.any(String),
+      hash: expect.any(Buffer),
       outputs: expect.arrayContaining([
         expect.objectContaining({
-          note: expect.any(String),
+          note: expect.any(Buffer),
         }),
       ]),
     }),

src/upload/index.test.ts

@@ -1,68 +1,36 @@
 import fs from "fs";
 import os from "os";
 import path from "path";
+import { open } from "fs/promises";
 
 import { createGunzip } from "zlib";
 import { lightBlockCache } from "../cache";
 import { lightBlockUpload } from "./index";
-import {
-  PutObjectCommand,
-  ListObjectsV2Command,
-  S3Client,
-} from "@aws-sdk/client-s3";
 import { LightBlock } from "../models/lightstreamer";
+import { createInterface } from "readline";
 
 describe("LightBlockUpload", () => {
-  beforeAll(() => {
-    jest.spyOn(S3Client.prototype, "send").mockImplementation((command) => {
-      if (command instanceof ListObjectsV2Command) {
-        return Promise.resolve({
-          Contents: [
-            { Key: lightBlockUpload.uploadName({ start: 1, end: 1000 }) },
-            { Key: lightBlockUpload.uploadName({ start: 1001, end: 2000 }) },
-          ],
-        });
-      } else if (command instanceof PutObjectCommand) {
-        return Promise.resolve({
-          /* your mock PutObjectCommand response */
-        });
-      } else {
-        throw new Error(
-          `Command mock not implemented: ${command.constructor.name}`,
-        );
-      }
-    });
-  });
-
   afterAll(async () => {
     jest.resetAllMocks();
     await lightBlockCache.close();
   });
 
-  it("upload name creation should be reversible", () => {
-    const blockRange = { start: 1, end: 1000 };
-    const key = lightBlockUpload.uploadName(blockRange);
-    const newBlockRange = lightBlockUpload.parseUploadName(key);
-    expect(blockRange).toEqual(newBlockRange);
-  });
-
-  it("existing uploads should return block ranges", async () => {
-    const ranges = await lightBlockUpload.existingUploads();
-    expect(ranges).toEqual([
-      { start: 1, end: 1000 },
-      { start: 1001, end: 2000 },
-    ]);
-  });
+  it("should gzip blocks/manifest, last block should match", async () => {
+    const tempFile = path.join(os.tmpdir(), "test");
+    const blockFile = await lightBlockUpload.createBlockFiles(tempFile, 0);
 
-  it("should gzip blocks as expected", async () => {
-    const tempFile = path.join(os.tmpdir(), "test.gz");
-    await lightBlockUpload.gzipBlocks(1, 0.001, tempFile);
+    const tempGz = path.join(os.tmpdir(), "test.gz");
+    const tempManifestGz = path.join(os.tmpdir(), "test.manifest.gz");
+    await lightBlockUpload.gzipFile(blockFile.file, tempGz);
+    await lightBlockUpload.gzipFile(blockFile.manifest, tempManifestGz);
 
     const gunzip = createGunzip();
-    const inputFile = fs.createReadStream(tempFile);
-
+    const inputFile = fs.createReadStream(tempGz);
     inputFile.pipe(gunzip);
 
+    let lastBlock: LightBlock | undefined;
+
+    // Verify unzipped data is correct
     let leftover = Buffer.alloc(0);
     gunzip.on("data", (chunk) => {
       let data = Buffer.concat([leftover, chunk]);
@@ -71,7 +39,7 @@ describe("LightBlockUpload", () => {
         const blockLength = data.readUInt32BE(0);
         if (data.length >= 4 + blockLength) {
           const blockData = data.subarray(4, 4 + blockLength);
-          expect(() => LightBlock.decode(blockData)).not.toThrow();
+          lastBlock = LightBlock.decode(blockData);
           data = data.subarray(4 + blockLength);
         } else {
           break;
@@ -90,5 +58,38 @@ describe("LightBlockUpload", () => {
         throw new Error(err.message);
       }).not.toThrow();
     });
+
+    // Now get blocks via the manifest and raw file
+    const tempGunzipped = path.join(os.tmpdir(), "test-gunzipped");
+    let lastManifestLine: string | undefined;
+    await new Promise((resolve) => {
+      fs.createReadStream(tempManifestGz)
+        .pipe(createGunzip())
+        .pipe(fs.createWriteStream(tempGunzipped))
+        .on("finish", resolve);
+    });
+
+    const rl = createInterface({
+      input: fs.createReadStream(tempGunzipped),
+      output: process.stdout,
+      terminal: false,
+    });
+
+    for await (const line of rl) {
+      lastManifestLine = line;
+    }
+    const splitLastLine = lastManifestLine!.split(",");
+    const byteStart = parseInt(splitLastLine![1]);
+    const byteEnd = parseInt(splitLastLine![2]);
+
+    const fileDescriptor = await open(tempFile, "r");
+    const buffer = Buffer.alloc(byteEnd - byteStart + 1);
+    await fileDescriptor.read(buffer, 0, byteEnd - byteStart + 1, byteStart);
+    console.log("biuff", buffer.toString("hex"));
+    const lastBlockManifest = LightBlock.decode(buffer);
+
+    // verify block info gotten from binary/manifest is same as gzip
+    expect(lastBlock?.sequence).toEqual(lastBlockManifest?.sequence);
+    expect(lastBlock?.hash).toEqual(lastBlockManifest?.hash);
   });
 });