Virtual IO, Cache (#72)

* fake io

* rename to virtual

* forest get

* cache

* fix cli

* test virtual io

* remove log

cli.mjs

@@ -14,5 +14,5 @@ if (args.length < 2) {
 }
 
 const hostName = args[2]
-const getFunc = hostName === undefined ? getLocal : getRemote(hostName)
+const getFunc = hostName === undefined ? getLocal({}) : getRemote({})(hostName)
 getFunc(node)([args[0], args[1]])

forest/index.mjs

@@ -55,9 +55,13 @@ const { tailToNodeId } = nodeId
  * @typedef { ForestNodeState[] } State
 */
 
+/**
+ * @typedef {(forestNodeId: ForestNodeId) => Promise<Uint8Array>} ForestGet
+*/
+
 /**
  * @typedef {{
- * readonly read: (address: ForestNodeId) => Promise<Uint8Array>,
+ * readonly read: ForestGet,
  * readonly write: (buffer: Uint8Array) => Promise<void>,
  * }} Provider
 */

index.mjs

@@ -1,12 +1,17 @@
+import nodeId from './cdt/node-id.mjs'
+import forest from './forest/index.mjs'
 import getModule from './forest/index.mjs'
 /** @typedef {import('./cdt/main-tree.mjs').State} StateTree */
 /**
  * @template T
  * @typedef {import('./cdt/sub-tree.mjs').Nullable<T>} Nullable
  */
 /** @typedef {import('./forest/index.mjs').ForestNodeId} ForestNodeId */
+/** @typedef {import('./forest/index.mjs').ForestGet} ForestGet */
 /** @typedef {import('./io/io.mjs').IO} IO */
 
+/** @typedef {{[index: string] : Uint8Array | undefined}} Cache */
+
 const { get } = getModule
 
 /** @type {(forestNodeId: ForestNodeId) => string} */
@@ -17,14 +22,28 @@ const getPath = ([forestNodeId, isRoot]) => {
 
 /** @type {(hostName: string) => (io: IO) => (forestNodeId: ForestNodeId) => Promise<Uint8Array>} */
 const fetchRead = hostName => ({ fetch }) => forestNodeId => fetch(`https://${hostName}/${getPath(forestNodeId)}`)
-    .then(async (resp) => resp.arrayBuffer().then(buffer => new Uint8Array(buffer)))
+  .then(async (resp) => resp.arrayBuffer().then(buffer => new Uint8Array(buffer)))
+
+/** @type {(mem: Cache) => (forestGet: ForestGet) => ForestGet} */
+const cache = mem => forestGet => {
+  return async (nodeId) => {
+    const nodeIdString = `${nodeId[0]}${nodeId[1]}`
+    let buffer = mem[nodeIdString]
+    if (buffer !== undefined) {
+      return buffer
+    }
+    buffer = await forestGet(nodeId)
+    mem[nodeIdString] = buffer
+    return buffer
+  }
+}
 
-/** @type {(io: IO) => (root: [string, string]) => Promise<number>} */
-const getLocal = io => async ([root, file]) => {
+/** @type {(mem: Cache) => (io: IO) => (root: [string, string]) => Promise<number>} */
+const getLocal = mem => io => async ([root, file]) => {
   const tempFile = `_temp_${root}`
   await io.write(tempFile, new Uint8Array())
-  /** @type {(forestNodeId: ForestNodeId) => Promise<Uint8Array>} */
-  const read = forestNodeId => io.read(getPath(forestNodeId))
+  /** @type {ForestGet} */
+  const read = cache(mem)(forestNodeId => io.read(getPath(forestNodeId)))
   /** @type {(buffer: Uint8Array) => Promise<void>} */
   const write = buffer => io.append(tempFile, buffer)
   const error = await get({ read, write })(root)
@@ -36,12 +55,12 @@ const getLocal = io => async ([root, file]) => {
   return 0
 }
 
-/** @type {(host: string) => (io: IO) => (root: [string, string]) => Promise<number>} */
-const getRemote = host => io => async ([root, file]) => {
+/** @type {(mem: Cache) => (host: string) => (io: IO) => (root: [string, string]) => Promise<number>} */
+const getRemote = mem => host => io => async ([root, file]) => {
   const tempFile = `_temp_${root}`
   await io.write(tempFile, new Uint8Array())
-  /** @type {(forestNodeId: ForestNodeId) => Promise<Uint8Array>} */
-  const read = fetchRead(host)(io)
+  /** @type {ForestGet} */
+  const read = cache(mem)(fetchRead(host)(io))
   /** @type {(buffer: Uint8Array) => Promise<void>} */
   const write = buffer => io.append(tempFile, buffer)
   const error = await get({ read, write })(root)

io/virtual.mjs

@@ -0,0 +1,56 @@
+/** @typedef {import('./io.mjs').IO} IO */
+
+const notImplemented = () => { throw 'not implemented' }
+
+/**
+ * @typedef {{[index: string]: Uint8Array}} FileSystem
+ */
+
+/** @type {(fs: FileSystem) => (path: string) => Promise<Uint8Array>} */
+const read = fs => async (path) => {
+  const buffer = fs[path]
+  if (buffer === undefined) {
+    throw 'file not found'
+  }
+  return buffer
+}
+
+/** @type {(fs: FileSystem) => (path: string, buffer: Uint8Array) => Promise<void>} */
+const append = fs => async (path, buffer) => {
+  const cur = fs[path]
+  if (buffer === undefined) {
+    throw 'file not found'
+  }
+  fs[path] = new Uint8Array([...cur, ...buffer])
+}
+
+/** @type {(fs: FileSystem) => (path: string, buffer: Uint8Array) => Promise<void>} */
+const write = fs => async (path, buffer) => {
+  fs[path] = buffer
+}
+
+/** @type {(fs: FileSystem) => (oldPath: string, newPath: string) => Promise<void>} */
+const rename = fs => async (oldPath, newPath) => {
+  const buffer = fs[oldPath]
+  if (buffer === undefined) {
+    throw 'file not found'
+  }
+  delete fs[oldPath]
+  fs[newPath] = buffer
+}
+
+/** @type {(fs: FileSystem) => IO} */
+const virtual = fs => {
+  return {
+    read: read(fs),
+    append: append(fs),
+    write: write(fs),
+    rename: rename(fs),
+    fetch: notImplemented,
+    document: undefined
+  }
+}
+
+export default {
+  virtual
+}

test.mjs

@@ -7,16 +7,20 @@ import index from './index.mjs'
 import ioNode from './io/node.mjs'
 import fs from 'node:fs'
 import fsPromises from 'node:fs/promises'
+import ioVirtual from './io/virtual.mjs'
 /** @typedef {import('./cdt/sub-tree.mjs').State} StateSubTree */
 /** @typedef {import('./cdt/main-tree.mjs').State} StateTree */
 /** @typedef {import('./io/io.mjs').IO} IO */
+/** @typedef {import('./io/virtual.mjs').FileSystem} FileSystem */
+/** @typedef {import('./index.mjs').Cache} Cache */
 const { toBase32Hash, getParityBit } = base32
 const { compress } = sha224
 const { merge, byteToNodeId, len } = nodeId
 const { highestOne256, height, push: pushSubTree } = subTree
 const { push: pushTree, end: endTree } = mainTree
 const { getLocal, getRemote } = index
 const { node, nodeSync } = ioNode
+const { virtual } = ioVirtual
 
 console.log(`test start`)
 
@@ -201,6 +205,30 @@ console.log(`test start`)
   }
 }
 
+const virtualFsTest = async () => {
+  /** @type {FileSystem} */
+  const fs = {}
+  const io = virtual(fs)
+  await io.write('test', new Uint8Array([0, 1, 2]))
+  let buffer = await io.read('test')
+  if (buffer.toString() !== '0,1,2') { throw buffer }
+
+  await io.write('test', new Uint8Array([3, 4, 5]))
+  buffer = await io.read('test')
+  if (buffer.toString() !== '3,4,5') { throw buffer }
+
+  await io.append('test', new Uint8Array([6, 7, 8]))
+  buffer = await io.read('test')
+  if (buffer.toString() !== '3,4,5,6,7,8') { throw buffer }
+
+  await io.rename('test', 'test-new')
+  //buffer = await io.read('test') //catch error
+  buffer = await io.read('test-new')
+  if (buffer.toString() !== '3,4,5,6,7,8') { throw buffer }
+}
+
+virtualFsTest()
+
 {
   const data = fs.readFileSync(`examples/small.txt`)
   /** @type {StateTree} */
@@ -271,12 +299,14 @@ const runTestsGet = io => async (getFunc) => {
 }
 
 const mainTestAsync = async () => {
+  /** @type {Cache} */
+  const mem = {}
   console.log('sync provider')
-  await runTestsGet(nodeSync)(getLocal)
+  await runTestsGet(nodeSync)(getLocal(mem))
   console.log('async provider')
-  await runTestsGet(node)(getLocal)
+  await runTestsGet(node)(getLocal(mem))
   console.log('fetch provider')
-  await runTestsGet(node)(getRemote('410f5a49.blockset-js-test.pages.dev'))
+  await runTestsGet(node)(getRemote({})('410f5a49.blockset-js-test.pages.dev'))
 }
 
 mainTestAsync()