From 0478813f06d2e4be3e5decfd284ea3f353164ecb Mon Sep 17 00:00:00 2001 From: Steven Vandevelde Date: Sun, 18 Feb 2024 22:42:04 +0100 Subject: [PATCH] feat: add ability to read with offset and length + update to wnfs 0.2.0 --- packages/nest/package.json | 2 +- packages/nest/src/class.ts | 36 +++++++++----- packages/nest/src/mutations.ts | 5 +- packages/nest/src/queries.ts | 82 +++++++++++++++++++------------- packages/nest/src/references.ts | 7 ++- packages/nest/src/store.ts | 18 +++---- packages/nest/src/transaction.ts | 16 +++++-- packages/nest/src/unix.ts | 2 +- packages/nest/test/class.test.ts | 73 ++++++++++++++++++++++++++-- pnpm-lock.yaml | 10 ++-- 10 files changed, 181 insertions(+), 70 deletions(-) diff --git a/packages/nest/package.json b/packages/nest/package.json index a57c9ba..e4c9f9e 100644 --- a/packages/nest/package.json +++ b/packages/nest/package.json @@ -117,7 +117,7 @@ "multiformats": "^12.1.3", "p-debounce": "^4.0.0", "uint8arrays": "^5.0.1", - "wnfs": "0.1.27" + "wnfs": "0.2.0" }, "devDependencies": { "@types/assert": "^1.5.9", diff --git a/packages/nest/src/class.ts b/packages/nest/src/class.ts index 283bbba..a3bd3d8 100644 --- a/packages/nest/src/class.ts +++ b/packages/nest/src/class.ts @@ -1,5 +1,10 @@ import type { Blockstore } from 'interface-blockstore' -import type { PrivateForest, PublicDirectory, PublicFile } from 'wnfs' +import type { + PrivateForest, + PublicDirectory, + PublicFile, + PublicNode, +} from 'wnfs' import { CID } from 'multiformats/cid' import { AccessKey, PrivateDirectory, PrivateFile, PrivateNode } from 'wnfs' @@ -401,7 +406,7 @@ export class FileSystem { capsuleKey: Uint8Array }, dataType: D, - options?: { offset: number; length: number } + options?: { offset?: number; length?: number } ): Promise> async read( path: @@ -412,7 +417,7 @@ export class FileSystem { capsuleKey: Uint8Array }, dataType: DataType, - options?: { offset: number; length: number } + options?: { offset?: number; length?: number } ): Promise> { return await this.#transactionContext().read( path, @@ -722,25 +727,34 @@ export class FileSystem { switch (partition.name) { case 'public': { - const node = + const wnfsBlockstore = Store.wnfs(this.#blockstore) + + const node: PublicNode | null | undefined = partition.segments.length === 0 ? this.#rootTree.publicRoot().asNode() : await this.#rootTree .publicRoot() - .getNode(partition.segments, Store.wnfs(this.#blockstore)) + .getNode(partition.segments, wnfsBlockstore) + if (node === null || node === undefined) throw new Error('Failed to find needed public node for infusion') - const fileOrDir: PublicFile | PublicDirectory = - node.isFile() === true ? node.asFile() : node.asDir() + const fileOrDir: PublicFile | PublicDirectory = node.isFile() + ? node.asFile() + : node.asDir() const capsuleCID = await fileOrDir .store(Store.wnfs(this.#blockstore)) .then((a) => CID.decode(a as Uint8Array)) - const contentCID = - node.isFile() === true - ? CID.decode(node.asFile().contentCid() as Uint8Array) - : capsuleCID + + const contentCID = node.isFile() + ? CID.decode( + await node + .asFile() + .getRawContentCid(wnfsBlockstore) + .then((u) => u as Uint8Array) + ) + : capsuleCID return { dataRoot, diff --git a/packages/nest/src/mutations.ts b/packages/nest/src/mutations.ts index 86d0444..448e17b 100644 --- a/packages/nest/src/mutations.ts +++ b/packages/nest/src/mutations.ts @@ -12,7 +12,6 @@ import type { } from './types/internal.js' import * as Store from './store.js' -import * as Unix from './unix.js' import { searchLatest } from './common.js' @@ -51,13 +50,11 @@ export const publicRemove = () => { export const publicWrite = (bytes: Uint8Array) => { return async (params: PublicParams): Promise => { - const cid = await Unix.importFile(bytes, params.blockstore) - return await params.rootTree .publicRoot() .write( params.pathSegments, - cid.bytes, + bytes, new Date(), Store.wnfs(params.blockstore) ) diff --git a/packages/nest/src/queries.ts b/packages/nest/src/queries.ts index 87b297c..2d4607c 100644 --- a/packages/nest/src/queries.ts +++ b/packages/nest/src/queries.ts @@ -1,3 +1,4 @@ +import type { CID } from 'multiformats/cid' import type { Blockstore } from 'interface-blockstore' import type { AccessKey, @@ -8,7 +9,6 @@ import type { } from 'wnfs' import { PrivateNode } from 'wnfs' -import { CID } from 'multiformats/cid' import * as Store from './store.js' import * as Path from './path.js' @@ -103,22 +103,33 @@ export const publicListDirectoryWithKind = () => { } } -export const publicRead = (options?: { offset: number; length: number }) => { +export const publicRead = (options?: { offset?: number; length?: number }) => { return async (params: PublicParams): Promise => { - const result = await params.rootTree + const wnfsBlockStore = Store.wnfs(params.blockstore) + + const node: PublicNode | null | undefined = await params.rootTree .publicRoot() - .read(params.pathSegments, Store.wnfs(params.blockstore)) + .getNode(params.pathSegments, wnfsBlockStore) - return await publicReadFromCID( - CID.decode(result as Uint8Array), - options - )(params) + if (node === null || node === undefined) { + throw new Error('Failed to find public node') + } else if (node.isDir()) { + throw new Error('Expected node to be a file') + } + + return await node + .asFile() + .readAt( + options?.offset ?? 0, + options?.length ?? undefined, + wnfsBlockStore + ) } } export const publicReadFromCID = ( cid: CID, - options?: { offset: number; length: number } + options?: { offset?: number; length?: number } ) => { return async (context: PublicContext): Promise => { return await Unix.exportFile(cid, context.blockstore, options) @@ -243,46 +254,51 @@ export const privateListDirectoryWithKind = () => { } } -export const privateRead = (_options?: { offset: number; length: number }) => { +export const privateRead = (options?: { offset?: number; length?: number }) => { return async (params: PrivateParams): Promise => { - // TODO: Respect `offset` and `length` options when private streaming API is exposed in rs-wnfs - // const offset = options?.offset - // const length = options?.length + let node - let bytes + if (params.node.isDir()) { + if (params.remainder.length === 0) { + throw new Error('Expected node to be a file') + } - if (params.node.isFile()) { - bytes = await params.node - .asFile() - .getContent( - params.rootTree.privateForest(), - Store.wnfs(params.blockstore) - ) - } else { - const { result } = await params.node + const tmpNode: PrivateNode | null | undefined = await params.node .asDir() - .read( + .getNode( params.remainder, searchLatest(), params.rootTree.privateForest(), Store.wnfs(params.blockstore) ) - bytes = result + + if (tmpNode === null || tmpNode === undefined) { + throw new Error('Failed to find private node') + } else if (tmpNode.isDir()) { + throw new Error('Expected node to be a file') + } + + node = tmpNode + } else { + node = params.node } - return bytes + return await node + .asFile() + .readAt( + options?.offset ?? 0, + options?.length ?? undefined, + params.rootTree.privateForest(), + Store.wnfs(params.blockstore) + ) } } export const privateReadFromAccessKey = ( accessKey: AccessKey, - _options?: { offset: number; length: number } + options?: { offset?: number; length?: number } ) => { return async (context: PrivateContext): Promise => { - // TODO: Respect `offset` and `length` options when private streaming API is exposed in rs-wnfs - // const offset = options?.offset - // const length = options?.length - // Retrieve node const node = await PrivateNode.load( accessKey, @@ -294,7 +310,9 @@ export const privateReadFromAccessKey = ( const file: PrivateFile = node.asFile() // TODO: Respect the offset and length options when available in rs-wnfs - return await file.getContent( + return await file.readAt( + options?.offset ?? 0, + options?.length ?? undefined, context.rootTree.privateForest(), Store.wnfs(context.blockstore) ) diff --git a/packages/nest/src/references.ts b/packages/nest/src/references.ts index 0a2c3f8..ed3c436 100644 --- a/packages/nest/src/references.ts +++ b/packages/nest/src/references.ts @@ -21,7 +21,12 @@ export async function contentCID( const maybeNode: PublicNode | undefined = result ?? undefined return maybeNode?.isFile() === true - ? CID.decode(maybeNode.asFile().contentCid()) + ? CID.decode( + await maybeNode + .asFile() + .getRawContentCid(wnfsBlockstore) + .then((u) => u as Uint8Array) + ) : undefined } diff --git a/packages/nest/src/store.ts b/packages/nest/src/store.ts index c521d22..22d6e80 100644 --- a/packages/nest/src/store.ts +++ b/packages/nest/src/store.ts @@ -1,14 +1,12 @@ import type { Blockstore } from 'interface-blockstore' +import type { BlockStore as WnfsBlockStore } from 'wnfs' import { CID } from 'multiformats/cid' import { sha256 } from 'multiformats/hashes/sha2' // 🧩 -export interface WnfsBlockStore { - getBlock: (cid: Uint8Array) => Promise - putBlock: (bytes: Uint8Array, code: number) => Promise -} +export type { BlockStore as WnfsBlockStore } from 'wnfs' // 🛠️ @@ -24,10 +22,14 @@ export function wnfs(blockstore: Blockstore): WnfsBlockStore { return await blockstore.get(decodedCid) }, - async putBlock(bytes: Uint8Array, code: number): Promise { - const c = await cid(bytes, code) - await blockstore.put(c, bytes) - return c.bytes + async hasBlock(cid: Uint8Array): Promise { + const decodedCid = CID.decode(cid) + return await blockstore.has(decodedCid) + }, + + async putBlockKeyed(cid: Uint8Array, bytes: Uint8Array): Promise { + const decodedCid = CID.decode(cid) + await blockstore.put(decodedCid, bytes) }, } } diff --git a/packages/nest/src/transaction.ts b/packages/nest/src/transaction.ts index 504ee0b..acc1b35 100644 --- a/packages/nest/src/transaction.ts +++ b/packages/nest/src/transaction.ts @@ -246,7 +246,7 @@ export class TransactionContext { capsuleKey: Uint8Array }, dataType: DataType, - options?: { offset: number; length: number } + options?: { offset?: number; length?: number } ): Promise> async read( arg: @@ -257,7 +257,7 @@ export class TransactionContext { capsuleKey: Uint8Array }, dataType: DataType, - options?: { offset: number; length: number } + options?: { offset?: number; length?: number } ): Promise> { let bytes @@ -268,14 +268,22 @@ export class TransactionContext { options )(this.#publicContext()) } else if ('capsuleCID' in arg) { + const wnfsBlockstore = Store.wnfs(this.#blockstore) + // Public content from capsule CID const publicFile: PublicFile = await PublicFile.load( arg.capsuleCID.bytes, - Store.wnfs(this.#blockstore) + wnfsBlockstore ) return await this.read( - { contentCID: CID.decode(publicFile.contentCid()) }, + { + contentCID: CID.decode( + await publicFile + .getRawContentCid(wnfsBlockstore) + .then((u) => u as Uint8Array) + ), + }, dataType, options ) diff --git a/packages/nest/src/unix.ts b/packages/nest/src/unix.ts index 270d52d..8bf2dcb 100644 --- a/packages/nest/src/unix.ts +++ b/packages/nest/src/unix.ts @@ -33,7 +33,7 @@ export function createDirectory( export async function exportFile( cid: CID, store: Blockstore, - options?: { offset: number; length: number } + options?: { offset?: number; length?: number } ): Promise { const offset = options?.offset const length = options?.length diff --git a/packages/nest/test/class.test.ts b/packages/nest/test/class.test.ts index 887b990..449fa12 100644 --- a/packages/nest/test/class.test.ts +++ b/packages/nest/test/class.test.ts @@ -8,7 +8,6 @@ import type { CID } from 'multiformats' import { MemoryBlockstore } from 'blockstore-core/memory' import * as Path from '../src/path.js' -import * as Unix from '../src/unix.js' import type { Modification } from '../src/types.js' import { FileSystem } from '../src/class.js' @@ -52,14 +51,14 @@ describe('File System Class', () => { const publicPath = Path.file('public', 'nested-public', 'public.txt') const privatePath = Path.file('private', 'nested-private', 'private.txt') - const { contentCID } = await fs.write(publicPath, 'utf8', 'public') + await fs.write(publicPath, 'utf8', 'public') const { capsuleKey, dataRoot } = await fs.write( privatePath, 'utf8', 'private' ) - const contentBytes = await Unix.exportFile(contentCID, blockstore) + const contentBytes = await fs.read(publicPath, 'bytes') assert.equal(new TextDecoder().decode(contentBytes), 'public') @@ -226,6 +225,74 @@ describe('File System Class', () => { assert.equal(await fs.read({ capsuleKey }, 'utf8'), '🔐') }) + it('can read partial public content bytes', async () => { + const { contentCID, capsuleCID } = await fs.write( + Path.file('public', 'file'), + 'bytes', + new Uint8Array([16, 24, 32]) + ) + + assert.equal( + await fs + .read({ contentCID }, 'bytes', { offset: 1 }) + .then((a) => a.toString()), + new Uint8Array([24, 32]).toString() + ) + + assert.equal( + await fs + .read({ capsuleCID }, 'bytes', { offset: 1 }) + .then((a) => a.toString()), + new Uint8Array([24, 32]).toString() + ) + }) + + it('can read partial utf8 public content', async () => { + const { contentCID, capsuleCID } = await fs.write( + Path.file('public', 'file'), + 'utf8', + 'abc' + ) + + assert.equal( + await fs.read({ contentCID }, 'utf8', { offset: 1, length: 1 }), + 'b' + ) + + assert.equal( + await fs.read({ capsuleCID }, 'utf8', { offset: 1, length: 1 }), + 'b' + ) + }) + + it('can read partial private content bytes', async () => { + const { capsuleKey } = await fs.write( + Path.file('private', 'file'), + 'bytes', + new Uint8Array([16, 24, 32]) + ) + + assert.equal( + await fs + .read({ capsuleKey }, 'bytes', { offset: 1 }) + .then((a) => a.toString()), + new Uint8Array([24, 32]).toString() + ) + }) + + it('can read partial utf8 private content', async () => { + const { capsuleKey } = await fs.write( + Path.file('private', 'file'), + 'utf8', + 'abc' + ) + + assert.equal( + await fs.read({ capsuleKey }, 'utf8', { offset: 1, length: 1 }), + 'b' + ) + }) + // DIRECTORIES // ----------- diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 056fa1f..1584759 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,4 +1,4 @@ -lockfileVersion: '6.1' +lockfileVersion: '6.0' settings: autoInstallPeers: true @@ -103,8 +103,8 @@ importers: specifier: ^5.0.1 version: 5.0.1 wnfs: - specifier: 0.1.27 - version: 0.1.27 + specifier: 0.2.0 + version: 0.2.0 devDependencies: '@types/assert': specifier: ^1.5.9 @@ -5685,8 +5685,8 @@ packages: isexe: 2.0.0 dev: true - /wnfs@0.1.27: - resolution: {integrity: sha512-9cC53nL3Nd303gykf6WNLYoxBsGkR9sgG2PMLMRcaMD3Ygzd6Q9sUFz7dPhYNWUPIH9VUGLM58MA16c03HrX0Q==} + /wnfs@0.2.0: + resolution: {integrity: sha512-K2uXtk8JyHRSLC0wwe+WfHDxDWnwyc5obmCZlMLbkuqG+Bu1OxsdQeL5tzd90kDjRl7YbdWMANR/I0IS1pgqAQ==} dev: false /workerpool@6.2.1: