diff --git a/package-lock.json b/package-lock.json index f5c3034d..4a056983 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "geotiff": "^2.0.5", "three": "^0.144.0", "tweakpane": "^3.1.9", - "zarr": "^0.5.2" + "zarrita": "^0.3.2" }, "devDependencies": { "@babel/cli": "^7.14.8", @@ -3128,6 +3128,40 @@ "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", "dev": true }, + "node_modules/@zarrita/core": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@zarrita/core/-/core-0.0.3.tgz", + "integrity": "sha512-fWv51b+xbYnws1pkNDPwJQoDa76aojxplHyMup82u11UAiet3gURMsrrkhM6YbeTgSY1A8oGxDOrvar3SiZpLA==", + "dependencies": { + "@zarrita/storage": "^0.0.2", + "@zarrita/typedarray": "^0.0.1", + "numcodecs": "^0.2.2" + } + }, + "node_modules/@zarrita/indexing": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@zarrita/indexing/-/indexing-0.0.3.tgz", + "integrity": "sha512-Q61d9MYX6dsK1DLltEpwx4mJWCZHj0TXiaEN4QpxNDtToa/EoyytP/pYHPypO4GXBscZooJ6eZkKT5FMx9PVfg==", + "dependencies": { + "@zarrita/core": "^0.0.3", + "@zarrita/storage": "^0.0.2", + "@zarrita/typedarray": "^0.0.1" + } + }, + "node_modules/@zarrita/storage": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/@zarrita/storage/-/storage-0.0.2.tgz", + "integrity": "sha512-uFt4abAoiOYLroalNDAnVaQxA17zGKrQ0waYKmTVm+bNonz8ggKZP+0FqMhgUZITGChqoANHuYTazbuU5AFXWA==", + "dependencies": { + "reference-spec-reader": "^0.2.0", + "unzipit": "^1.3.6" + } + }, + "node_modules/@zarrita/typedarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/@zarrita/typedarray/-/typedarray-0.0.1.tgz", + "integrity": "sha512-ZdvNjYP1bEuQXoSTVkemV99w42jHYrJ3nh9golCLd4MVBlrVbfZo4wWgBslU4JZUaDikhFSH+GWMDgAq/rI32g==" + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -5551,7 +5585,8 @@ "node_modules/eventemitter3": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", - "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==" + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "dev": true }, "node_modules/events": { "version": "3.3.0", @@ -9795,21 +9830,6 @@ "node": ">=6" } }, - "node_modules/p-queue": { - "version": "7.3.4", - "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-7.3.4.tgz", - "integrity": "sha512-esox8CWt0j9EZECFvkFl2WNPat8LN4t7WWeXq73D9ha0V96qPRufApZi4ZhPwXAln1uVVal429HVVKPa2X0yQg==", - "dependencies": { - "eventemitter3": "^4.0.7", - "p-timeout": "^5.0.2" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/p-retry": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", @@ -9823,17 +9843,6 @@ "node": ">=8" } }, - "node_modules/p-timeout": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-5.1.0.tgz", - "integrity": "sha512-auFDyzzzGZZZdHz3BtET9VEz0SE/uMEAx7uWfGPucfzEwwe/xH0iVeZibQmANYE/hp9T2+UUZT5m+BKyrDp3Ew==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", @@ -10668,6 +10677,11 @@ "node": ">= 0.10" } }, + "node_modules/reference-spec-reader": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/reference-spec-reader/-/reference-spec-reader-0.2.0.tgz", + "integrity": "sha512-q0mfCi5yZSSHXpCyxjgQeaORq3tvDsxDyzaadA/5+AbAUwRyRuuTh0aRQuE/vAOt/qzzxidJ5iDeu1cLHaNBlQ==" + }, "node_modules/regenerate": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.2.tgz", @@ -12257,6 +12271,17 @@ "node": ">= 0.8" } }, + "node_modules/unzipit": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/unzipit/-/unzipit-1.4.3.tgz", + "integrity": "sha512-gsq2PdJIWWGhx5kcdWStvNWit9FVdTewm4SEG7gFskWs+XCVaULt9+BwuoBtJiRE8eo3L1IPAOrbByNLtLtIlg==", + "dependencies": { + "uzip-module": "^1.0.2" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/update-browserslist-db": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.11.tgz", @@ -12389,6 +12414,11 @@ "node": ">=8" } }, + "node_modules/uzip-module": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/uzip-module/-/uzip-module-1.0.3.tgz", + "integrity": "sha512-AMqwWZaknLM77G+VPYNZLEruMGWGzyigPK3/Whg99B3S6vGHuqsyl5ZrOv1UUF3paGK1U6PM0cnayioaryg/fA==" + }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", @@ -13246,16 +13276,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/zarr": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/zarr/-/zarr-0.5.2.tgz", - "integrity": "sha512-XiAZTlkCTALZyXCAXY2rgfRY45sIaGZd/rKKuQa84+bjpxoyNYXbAU5uaIDTR+CvIuTFABqq8Gc4PfzZHYOvkw==", + "node_modules/zarrita": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/zarrita/-/zarrita-0.3.2.tgz", + "integrity": "sha512-Zx9nS28C2tXZhF1BmQkgQGi0M/Z5JiM/KCMa+fEYtr/MnIzyizR4sKRA/sXjDP1iuylILWTJAWWBJD//0ONXCA==", "dependencies": { - "numcodecs": "^0.2.2", - "p-queue": "^7.1.0" - }, - "engines": { - "node": ">=12" + "@zarrita/core": "^0.0.3", + "@zarrita/indexing": "^0.0.3", + "@zarrita/storage": "^0.0.2" } }, "node_modules/zwitch": { diff --git a/package.json b/package.json index cfa35595..36af2fc1 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "geotiff": "^2.0.5", "three": "^0.144.0", "tweakpane": "^3.1.9", - "zarr": "^0.5.2" + "zarrita": "^0.3.2" }, "devDependencies": { "@babel/cli": "^7.14.8", diff --git a/src/loaders/OmeZarrLoader.ts b/src/loaders/OmeZarrLoader.ts index 5e283036..2aef5b08 100644 --- a/src/loaders/OmeZarrLoader.ts +++ b/src/loaders/OmeZarrLoader.ts @@ -1,8 +1,11 @@ import { Box3, Vector3 } from "three"; -import { HTTPStore, TypedArray, ZarrArray, openArray, openGroup, slice } from "zarr"; -import { RawArray } from "zarr/types/rawArray"; -import { AsyncStore } from "zarr/types/storage/types"; -import { Slice } from "zarr/types/core/types"; + +import * as zarr from "@zarrita/core"; +import { get as zarrGet, slice, Slice } from "@zarrita/indexing"; +import { AbsolutePath, AsyncReadable, Readable } from "@zarrita/storage"; +// Importing `FetchStore` from its home subpackage (@zarrita/storage) causes errors. +// Getting it from the top-level package means we don't get its type. This is also a bug, but it's more acceptable. +import { FetchStore } from "zarrita"; import Volume, { ImageInfo } from "../Volume"; import VolumeCache from "../VolumeCache"; @@ -88,91 +91,61 @@ type OMEZarrMetadata = { const getDimensionCount = ([t, c, z]: [number, number, number, number, number]) => 2 + Number(t > -1) + Number(c > -1) + Number(z > -1); +type WrappedStoreOpts = { + options?: Opts; + // TODO: store options common to a single chunk load here +}; + /** - * `Store` is zarr.js's minimal abstraction for anything that acts like a filesystem. (Local machine, HTTP server, etc.) - * `SmartStoreWrapper` wraps another `Store` and adds (optional) connections to `VolumeCache` and `RequestQueue`. - * - * NOTE: if using `RequestQueue`, *ensure that calls made on arrays using this store do not also do promise queueing* - * by setting the option `concurrencyLimit: Infinity`. + * `Readable` is zarrita's minimal abstraction for any source of data. + * `WrappedStore` wraps another `Readable` and adds (optional) connections to `VolumeCache` and `RequestQueue`. */ -class SmartStoreWrapper implements AsyncStore { - // Required by `AsyncStore` - listDir?: (path?: string) => Promise; - rmDir?: (path?: string) => Promise; - getSize?: (path?: string) => Promise; - rename?: (path?: string) => Promise; - - baseStore: AsyncStore; - +class WrappedStore = Readable> implements AsyncReadable> { + baseStore: S; cache?: VolumeCache; - requestQueue?: RequestQueue; + queue?: RequestQueue; - constructor(baseStore: AsyncStore, cache?: VolumeCache, requestQueue?: RequestQueue) { + constructor(baseStore: S, cache?: VolumeCache, queue?: RequestQueue) { this.baseStore = baseStore; this.cache = cache; - this.requestQueue = requestQueue; - this.listDir = baseStore.listDir; - this.rmDir = baseStore.rmDir; - this.getSize = baseStore.getSize; - this.rename = baseStore.rename; + this.queue = queue; } - private async getAndCacheItem(item: string, cacheKey: string, opts?: RequestInit): Promise { - const result = await this.baseStore.getItem(item, opts); - if (this.cache) { + private async getAndCache(key: AbsolutePath, cacheKey: string, opts?: Opts): Promise { + const result = await this.baseStore.get(key, opts); + if (this.cache && result) { this.cache.insert(cacheKey, result); } return result; } - getItem(item: string, opts?: RequestInit): Promise { - // If we don't have a cache or aren't getting a chunk, call straight to the base store - const zarrExts = [".zarray", ".zgroup", ".zattrs"]; - if (!this.cache || zarrExts.some((s) => item.endsWith(s))) { - return this.baseStore.getItem(item, opts); + async get(key: AbsolutePath, opts?: WrappedStoreOpts | undefined): Promise { + const ZARR_EXTS = [".zarray", ".zgroup", ".zattrs", "zarr.json"]; + if (!this.cache || ZARR_EXTS.some((s) => key.endsWith(s))) { + return this.baseStore.get(key, opts?.options); } - let keyPrefix = (this.baseStore as HTTPStore).url ?? ""; - if (keyPrefix !== "" && !keyPrefix.endsWith("/")) { + let keyPrefix = (this.baseStore as FetchStore).url ?? ""; + if (keyPrefix !== "" && !(keyPrefix instanceof URL) && !keyPrefix.endsWith("/")) { keyPrefix += "/"; } - const key = keyPrefix + item; + + const fullKey = keyPrefix + key.slice(1); // Check the cache - const cacheResult = this.cache.get(key); + const cacheResult = this.cache.get(fullKey); if (cacheResult) { - return Promise.resolve(cacheResult); + return new Uint8Array(cacheResult); } // Not in cache; load the chunk and cache it - if (this.requestQueue) { - return this.requestQueue.addRequest(key, () => this.getAndCacheItem(item, key, opts)); + if (this.queue) { + return this.queue.addRequest(fullKey, () => this.getAndCache(key, fullKey, opts?.options)); } else { // Should we ever hit this code? We should always have a request queue. - return this.getAndCacheItem(item, key, opts); + return this.getAndCache(key, fullKey, opts?.options); } } - - keys(): Promise { - return this.baseStore.keys(); - } - - async setItem(_item: string, _value: ArrayBuffer): Promise { - console.warn("zarr store wrapper: attempt to set data!"); - // return this.baseStore.setItem(item, value); - return false; - } - - async deleteItem(_item: string): Promise { - console.warn("zarr store wrapper: attempt to delete data!"); - // return this.baseStore.deleteItem(item); - return false; - } - - containsItem(item: string): Promise { - // zarr seems to never call this method on chunk paths (just .zarray, .zstore, etc.), so we don't check cache here - return this.baseStore.containsItem(item); - } } function remapAxesToTCZYX(axes: Axis[]): [number, number, number, number, number] { @@ -213,8 +186,8 @@ function pickLevelToLoad(loadSpec: LoadSpec, spatialDimsZYX: [number, number, nu return Math.max(optimalLevel, loadSpec.multiscaleLevel ?? 0); } -function convertChannel(channelData: TypedArray, dtype: string): Uint8Array { - if (dtype === "|u1") { +function convertChannel(channelData: zarr.TypedArray): Uint8Array { + if (channelData instanceof Uint8Array) { return channelData as Uint8Array; } @@ -242,8 +215,10 @@ function convertChannel(channelData: TypedArray, dtype: string): Uint8Array { return u8; } +type NumericZarrArray = zarr.Array; + class OMEZarrLoader implements IVolumeLoader { - scaleLevels: ZarrArray[]; + scaleLevels: NumericZarrArray[]; multiscaleMetadata: OMEMultiscale; omeroMetadata: OmeroTransitionalMetadata; axesTCZYX: [number, number, number, number, number]; @@ -257,7 +232,7 @@ class OMEZarrLoader implements IVolumeLoader { maxExtent?: Box3; private constructor( - scaleLevels: ZarrArray[], + scaleLevels: NumericZarrArray[], multiscaleMetadata: OMEMultiscale, omeroMetadata: OmeroTransitionalMetadata, axisTCZYX: [number, number, number, number, number], @@ -278,9 +253,10 @@ class OMEZarrLoader implements IVolumeLoader { ): Promise { // Setup: create queue and store, get basic metadata const queue = new RequestQueue(concurrencyLimit); - const store = new SmartStoreWrapper(new HTTPStore(url), cache, queue); - const group = await openGroup(store, null, "r"); - const metadata = (await group.attrs.asObject()) as OMEZarrMetadata; + const store = new WrappedStore(new FetchStore(url), cache, queue); + const root = zarr.root(store); + const group = await zarr.open(root, { kind: "group" }); + const metadata = group.attrs as OMEZarrMetadata; // Pick scene (multiscale) if (scene > metadata.multiscales.length) { @@ -290,12 +266,11 @@ class OMEZarrLoader implements IVolumeLoader { const multiscale = metadata.multiscales[scene]; // Open all scale levels of multiscale - const scaleLevelPaths = multiscale.datasets.map(({ path }) => path); - const scaleLevelPromises = scaleLevelPaths.map((path) => openArray({ store, path, mode: "r" })); + const scaleLevelPromises = multiscale.datasets.map(({ path }) => zarr.open(root.resolve(path), { kind: "array" })); const scaleLevels = await Promise.all(scaleLevelPromises); const axisTCZYX = remapAxesToTCZYX(multiscale.axes); - return new OMEZarrLoader(scaleLevels, multiscale, metadata.omero, axisTCZYX, queue); + return new OMEZarrLoader(scaleLevels as NumericZarrArray[], multiscale, metadata.omero, axisTCZYX, queue); } private getUnitSymbols(): [string, string] { @@ -504,8 +479,8 @@ class OMEZarrLoader implements IVolumeLoader { }); try { - const result = (await level.getRaw(sliceSpec, { concurrencyLimit: Infinity })) as RawArray; - const u8 = convertChannel(result.data, result.dtype); + const result = await zarrGet(level, sliceSpec); + const u8 = convertChannel(result.data); vol.setChannelDataFromVolume(ch, u8); onChannelLoaded?.(vol, ch); } catch (e) { diff --git a/src/test/RequestQueue.test.ts b/src/test/RequestQueue.test.ts index a6c3a2c7..be88aade 100644 --- a/src/test/RequestQueue.test.ts +++ b/src/test/RequestQueue.test.ts @@ -1,6 +1,6 @@ import { expect } from "chai"; import { Vector3 } from "three"; -import { TypedArray } from "zarr"; +import { TypedArray } from "@zarrita/core"; import RequestQueue, { Request } from "../utils/RequestQueue"; import { LoadSpec, loadSpecToString } from "../loaders/IVolumeLoader"; @@ -335,7 +335,7 @@ describe("test RequestQueue", () => { expect(count).to.equal(0); }); - async function mockLoader(loadSpec: Required, maxDelayMs = 10.0): Promise { + async function mockLoader(loadSpec: Required, maxDelayMs = 10.0): Promise> { const { x, y, z } = loadSpec.subregion.getSize(new Vector3()); const data = new Uint8Array(x * y * z); const delayMs = Math.random() * maxDelayMs;