feat: add basic writer (#3)

Co-authored-by: Tomás Ciccola <[email protected]>

package.json

@@ -5,33 +5,38 @@
   "version": "1.0.0",
   "main": "index.js",
   "scripts": {
-    "patch": "patch-package simple-hypercore-protocol",
+    "postinstall": "patch-package",
     "format": "prettier --write .",
     "test:prettier": "prettier --check .",
     "test:eslint": "eslint --cache .",
     "test:typescript": "tsc",
     "test:node": "node --test",
     "test": "npm-run-all --aggregate-output --print-label --parallel test:*",
-    "watch:test:typescript": "tsc --watch --project ./tsconfig.dev.json",
+    "watch:test:typescript": "npm run test:typescript -- --watch",
     "watch:test:node": "npm run test:node -- --watch"
   },
   "license": "MIT",
   "dependencies": {
+    "archiver": "^7.0.1",
     "hypercore-crypto": "^3.4.2",
     "multifeed": "^6.0.0",
-    "yazl": "^2.5.1"
+    "p-event": "^6.0.1"
   },
   "devDependencies": {
     "@eslint/js": "^9.15.0",
+    "@types/archiver": "^6.0.3",
     "@types/node": "^22.9.1",
+    "@types/yauzl-promise": "^4.0.1",
     "@types/yazl": "^2.4.5",
     "eslint": "^9.15.0",
     "globals": "^15.12.0",
     "npm-run-all": "^4.1.5",
     "patch-package": "^8.0.0",
     "prettier": "^3.3.3",
+    "tempy": "^3.1.0",
     "typescript": "^5.6.3",
-    "typescript-eslint": "^8.15.0"
+    "typescript-eslint": "^8.15.0",
+    "yauzl-promise": "^4.0.0"
   },
   "engines": {
     "node": "^22.11.0",

src/index.js

@@ -1 +1 @@
-export { default as MLEFWriter } from './writer.js'
+export { write } from './writer.js'

src/lib/hypercoreUtil.js

@@ -0,0 +1,58 @@
+/** @import { Hypercore } from 'hypercore' */
+
+/**
+ * Wraps `Hypercore.prototype.ready` in a promise.
+ *
+ * @param {Hypercore} hypercore
+ * @returns {Promise<void>}
+ */
+export const ready = (hypercore) =>
+  new Promise((resolve, reject) => {
+    hypercore.ready((err) => {
+      if (err) {
+        reject(err)
+      } else {
+        resolve()
+      }
+    })
+  })
+
+/**
+ * Wraps `Hypercore.prototype.rootHashes` in a promise.
+ *
+ * @param {Hypercore} hypercore
+ * @param {number} index
+ * @returns {Promise<Array<{
+ *   hash: Buffer
+ *   index: number
+ *   size: number
+ * }>>}
+ */
+export const rootHashes = (hypercore, index) =>
+  new Promise((resolve, reject) => {
+    hypercore.rootHashes(index, (err, roots) => {
+      if (err) {
+        return reject(err)
+      } else {
+        resolve(roots)
+      }
+    })
+  })
+
+/**
+ * Wraps `Hypercore.prototype.signature` in a promise.
+ *
+ * @param {Hypercore} hypercore
+ * @param {number} index
+ * @returns {Promise<{ index: number, signature: Buffer }>}
+ */
+export const signature = (hypercore, index) =>
+  new Promise((resolve, reject) => {
+    hypercore.signature(index, (err, signature) => {
+      if (err) {
+        reject(err)
+      } else {
+        resolve(signature)
+      }
+    })
+  })

src/lib/multifeedUtil.js

@@ -0,0 +1,15 @@
+/** @import { Multifeed } from 'multifeed' */
+
+/**
+ * @param {Multifeed} multifeed
+ * @returns {Promise<void>}
+ */
+export const ready = (multifeed) =>
+  new Promise((resolve, reject) => {
+    multifeed.once('error', reject)
+
+    multifeed.ready(() => {
+      multifeed.off('error', reject)
+      resolve()
+    })
+  })

src/lib/noop.js

@@ -0,0 +1 @@
+export const noop = () => {}

src/writer.js

@@ -1 +1,167 @@
-export default async function MLEFWriter() {}
+import archiver from 'archiver'
+import * as hypercoreCrypto from 'hypercore-crypto'
+import multifeed from 'multifeed'
+import fs from 'node:fs'
+import { readdir } from 'node:fs/promises'
+import * as path from 'node:path'
+import { pEvent } from 'p-event'
+import * as hypercoreUtil from './lib/hypercoreUtil.js'
+import * as multifeedUtil from './lib/multifeedUtil.js'
+import { noop } from './lib/noop.js'
+/** @import { Hypercore } from 'hypercore' */
+
+/**
+ * @param {string} inputPath path to `kappa.db` folder
+ * @param {string} outputPath path of file to save
+ * @returns {Promise<void>}
+ */
+export async function write(inputPath, outputPath) {
+  const output = fs.createWriteStream(outputPath)
+  const archive = archiver('zip', { zlib: { level: 9 } })
+
+  let throwArchiveErrorIfExists = noop
+  /** @param {Error} err */
+  const onArchiveError = (err) => {
+    throwArchiveErrorIfExists = () => {
+      throw err
+    }
+    archive.off('warning', onArchiveError)
+    archive.off('error', onArchiveError)
+  }
+  archive.once('warning', onArchiveError)
+  archive.once('error', onArchiveError)
+
+  archive.pipe(output)
+
+  throwArchiveErrorIfExists()
+
+  for await (const document of getInputDocuments(inputPath)) {
+    throwArchiveErrorIfExists()
+
+    const dirname = `docs/${document.id.slice(0, 2)}/${document.id}`
+    const basename = `${document.version || '_'}.json`
+    archive.append(JSON.stringify(document), {
+      name: `${dirname}/${basename}`,
+    })
+  }
+
+  const inputMediaPaths = await getInputMediaPaths(inputPath)
+  throwArchiveErrorIfExists()
+  for (const inputMediaPath of inputMediaPaths) {
+    archive.file(inputMediaPath, {
+      name: path.relative(inputPath, inputMediaPath),
+    })
+  }
+
+  const onOutputClosePromise = pEvent(output, 'close')
+  archive.finalize()
+  throwArchiveErrorIfExists()
+  await onOutputClosePromise
+}
+
+/**
+ * @typedef {object} HypercoreMetadata
+ * @prop {string} rootHashChecksum
+ * @prop {string} signature
+ * @prop {string} coreKey
+ * @prop {number} blockIndex
+ */
+
+/**
+ * @typedef {object} InputDocument
+ * @prop {string} id
+ * @prop {null | string} version
+ * @prop {unknown} document
+ * @prop {HypercoreMetadata} hypercoreMetadata
+ */
+
+/**
+ * @param {string} inputPath
+ * @returns {AsyncGenerator<InputDocument>}
+ */
+async function* getInputDocuments(inputPath) {
+  const multi = multifeed(inputPath, {
+    createIfMissing: false,
+    valueEncoding: 'json',
+    stats: false,
+  })
+  await multifeedUtil.ready(multi)
+
+  for (const hypercore of multi.feeds()) {
+    await hypercoreUtil.ready(hypercore)
+    if (hypercore.length === 0) continue
+
+    const stream = hypercore.createReadStream()
+
+    const hypercoreMetadata = await getHypercoreMetadata(hypercore)
+
+    for await (const document of stream) {
+      const { id, version } = parseDocument(document)
+      yield { id, version, document, hypercoreMetadata }
+    }
+  }
+}
+
+/**
+ * @param {Hypercore} hypercore
+ * @returns {Promise<HypercoreMetadata>}
+ */
+async function getHypercoreMetadata(hypercore) {
+  const rootHashesPromise = hypercoreUtil.rootHashes(hypercore, 0)
+  const signaturePromise = hypercoreUtil.signature(
+    hypercore,
+    Math.max(0, hypercore.length - 1)
+  )
+  return {
+    rootHashChecksum: hypercoreCrypto
+      .tree(await rootHashesPromise)
+      .toString('hex'),
+    signature: (await signaturePromise).signature.toString('hex'),
+    coreKey: hypercore.key.toString('hex'),
+    blockIndex: hypercore.length,
+  }
+}
+
+/**
+ * @param {unknown} document
+ * @returns {{ id: string, version: null | string }}
+ */
+function parseDocument(document) {
+  if (typeof document !== 'object' || document === null) {
+    throw new Error('document is not an object')
+  }
+
+  if (!('id' in document) || typeof document.id !== 'string') {
+    throw new Error('document.id is not a string')
+  }
+
+  /** @type {null | string} */ let version = null
+  if (
+    'version' in document &&
+    document.version &&
+    typeof document.version === 'string'
+  ) {
+    version = document.version
+  }
+
+  return {
+    id: document.id,
+    version,
+  }
+}
+
+/**
+ * @param {string} inputPath
+ * @returns {Promise<Array<string>>}
+ */
+async function getInputMediaPaths(inputPath) {
+  const inputMediaRootPath = path.join(inputPath, 'media')
+  const inputMediaAllFiles = await readdir(inputMediaRootPath, {
+    recursive: true,
+    withFileTypes: true,
+  })
+  const inputMediaFiles = inputMediaAllFiles.filter((dirent) => dirent.isFile())
+  return inputMediaFiles.map((dirent) =>
+    path.join(dirent.parentPath, dirent.name)
+  )
+}

test/fixture/db1/0/key

@@ -0,0 +1 @@
+w�����䕗��ٰ���ɜ�a�T��5�_p

test/fixture/db1/0/localname

@@ -0,0 +1 @@
+default

test/fixture/db1/0/secret_key

@@ -0,0 +1 @@
+EJ�U�:�*@d�iF��^N]}ƽ`2����w�����䕗��ٰ���ɜ�a�T��5�_p

test/fixture/db1/1/data

@@ -0,0 +1,6 @@
+{"lon":-58.5067235,"lat":-34.5819385,"attachments":[],"tags":{"type":"craft","craft":"clay","categoryId":"clay","notes":"Nada"},"metadata":{"location":{"error":false,"permission":"granted","position":{"timestamp":1718822620160,"mocked":false,"coords":{"altitude":42.80000305175781,"heading":220.94805908203125,"altitudeAccuracy":2.857691764831543,"latitude":-34.5819385,"speed":0.4037386178970337,"longitude":-58.5067235,"accuracy":5.980000019073486}},"provider":{"backgroundModeEnabled":true,"gpsAvailable":true,"passiveAvailable":true,"locationServicesEnabled":true,"networkAvailable":true}}},"schemaVersion":3,"type":"observation","timestamp":"2024-06-19T18:43:40.490Z","created_at":"2024-06-19T18:43:40.487Z","id":"74cb70988a17b6e4","links":[]}
+{"lon":-58.5067677,"lat":-34.5819589,"attachments":[{"id":"4b04270213860542d3fb2d56275fe12f.jpg","type":"image/jpeg"}],"tags":{"type":"animal","categoryId":"animal","notes":"Un animal"},"metadata":{"location":{"error":false,"permission":"granted","position":{"timestamp":1718822970076,"mocked":false,"coords":{"altitude":42.400001525878906,"heading":0,"altitudeAccuracy":2.6408803462982178,"latitude":-34.5819589,"speed":0.00892753154039383,"longitude":-58.5067677,"accuracy":3.9000000953674316}},"provider":{"backgroundModeEnabled":true,"gpsAvailable":true,"passiveAvailable":true,"locationServicesEnabled":true,"networkAvailable":true}}},"schemaVersion":3,"type":"observation","timestamp":"2024-06-19T18:49:43.200Z","created_at":"2024-06-19T18:49:43.199Z","id":"e670d3ce4a611413","links":[]}
+{"lon":-58.5067985,"lat":-34.5819491,"attachments":[],"tags":{"type":"activity","activity":"gathering","categoryId":"gathering-site","notes":"Esto ea otro"},"metadata":{"location":{"error":false,"permission":"granted","position":{"timestamp":1718823160077,"mocked":false,"coords":{"altitude":27.4,"heading":0,"altitudeAccuracy":26.200000762939453,"latitude":-34.5819491,"speed":0.0067694177851080894,"longitude":-58.5067985,"accuracy":3.9000000953674316}},"provider":{"backgroundModeEnabled":true,"gpsAvailable":true,"passiveAvailable":true,"locationServicesEnabled":true,"networkAvailable":true}}},"schemaVersion":3,"type":"observation","timestamp":"2024-06-19T18:52:48.317Z","created_at":"2024-06-19T18:52:48.317Z","id":"f40be305d4d190f1","links":[]}
+{"lon":-58.5067392,"lat":-34.5819587,"attachments":[{"id":"9e657b791646d5c8e1214c4b020a9dd3.jpg","type":"image/jpeg"}],"tags":{"place":"village","categoryId":"community"},"metadata":{"location":{"error":false,"permission":"granted","position":{"timestamp":1718823274077,"mocked":false,"coords":{"altitude":34.4,"heading":0,"altitudeAccuracy":20.899999618530273,"latitude":-34.5819587,"speed":0.001650038524530828,"longitude":-58.5067392,"accuracy":3.9000000953674316}},"provider":{"backgroundModeEnabled":true,"gpsAvailable":true,"passiveAvailable":true,"locationServicesEnabled":true,"networkAvailable":true}}},"schemaVersion":3,"type":"observation","timestamp":"2024-06-19T18:54:43.039Z","created_at":"2024-06-19T18:54:43.038Z","id":"d2b6e3d72200c082","links":[]}
+{"lon":-58.5067235,"lat":-34.5819385,"attachments":[],"tags":{"type":"craft","craft":"clay","categoryId":"clay","notes":"Nada de verdas posta"},"metadata":{"location":{"error":false,"permission":"granted","position":{"timestamp":1718822620160,"mocked":false,"coords":{"altitude":42.80000305175781,"heading":220.94805908203125,"altitudeAccuracy":2.857691764831543,"latitude":-34.5819385,"speed":0.4037386178970337,"longitude":-58.5067235,"accuracy":5.980000019073486}},"provider":{"backgroundModeEnabled":true,"gpsAvailable":true,"passiveAvailable":true,"locationServicesEnabled":true,"networkAvailable":true}}},"schemaVersion":3,"type":"observation","timestamp":"2024-06-19T19:02:44.829Z","created_at":"2024-06-19T18:43:40.487Z","id":"74cb70988a17b6e4","links":["54aef99c451b09d15f9a3e8f0059fe8d5a97ee2b2f0cb605e3f04153c5723012@0"],"version":"54aef99c451b09d15f9a3e8f0059fe8d5a97ee2b2f0cb605e3f04153c5723012@0","deviceId":"54aef99c451b09d15f9a3e8f0059fe8d5a97ee2b2f0cb605e3f04153c5723012"}
+{"lon":-58.5067235,"lat":-34.5819385,"attachments":[],"tags":{"type":"craft","craft":"clay","categoryId":"clay","notes":"Nada de verdas posta, pero esta vez de verdas"},"metadata":{"location":{"error":false,"permission":"granted","position":{"timestamp":1718822620160,"mocked":false,"coords":{"altitude":42.80000305175781,"heading":220.94805908203125,"altitudeAccuracy":2.857691764831543,"latitude":-34.5819385,"speed":0.4037386178970337,"longitude":-58.5067235,"accuracy":5.980000019073486}},"provider":{"backgroundModeEnabled":true,"gpsAvailable":true,"passiveAvailable":true,"locationServicesEnabled":true,"networkAvailable":true}}},"schemaVersion":3,"type":"observation","timestamp":"2024-06-19T19:24:05.792Z","created_at":"2024-06-19T18:43:40.487Z","id":"74cb70988a17b6e4","links":["54aef99c451b09d15f9a3e8f0059fe8d5a97ee2b2f0cb605e3f04153c5723012@4"],"version":"54aef99c451b09d15f9a3e8f0059fe8d5a97ee2b2f0cb605e3f04153c5723012@4","deviceId":"54aef99c451b09d15f9a3e8f0059fe8d5a97ee2b2f0cb605e3f04153c5723012"}

test/fixture/db1/bee80ff3a4ee5e727dc44197cb9d25bf8f19d50b0f3ad2984cfe5b7d14e75de7/key

@@ -0,0 +1 @@
+����^r}�A�˝%���:ҘL�[}�]�

test/fixture/db1/index/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000002

test/fixture/db1/index/LOG

@@ -0,0 +1 @@
+2024/06/19-15:41:09.762836 7b640e5496c0 Delete type=3 #1

test/fixture/db1/storage/meta

@@ -0,0 +1 @@
+{"bitfield":[],"branchFactor":4}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                

test/lib/noop.js

@@ -0,0 +1,7 @@
+import assert from 'node:assert/strict'
+import test from 'node:test'
+import { noop } from '../../src/lib/noop.js'
+
+test('returns undefined', () => {
+  assert.equal(noop(), undefined)
+})