diff --git a/public/index.ts b/public/index.ts index 875f171d..bbd03f9c 100644 --- a/public/index.ts +++ b/public/index.ts @@ -978,6 +978,9 @@ function createTestVolume() { timeScale: 1, timeUnit: "", + numMultiscaleLevels: 1, + multiscaleLevel: 0, + transform: { translation: new Vector3(0, 0, 0), rotation: new Vector3(0, 0, 0) }, }; diff --git a/src/Volume.ts b/src/Volume.ts index 9c2474d3..4493d403 100644 --- a/src/Volume.ts +++ b/src/Volume.ts @@ -4,12 +4,12 @@ import Channel from "./Channel"; import Histogram from "./Histogram"; import { getColorByChannelIndex } from "./constants/colors"; import { IVolumeLoader, LoadSpec, PerChannelCallback } from "./loaders/IVolumeLoader"; +import { estimateLevelForAtlas } from "./loaders/VolumeLoaderUtils"; export type ImageInfo = Readonly<{ name: string; /** XY size of the *original* (not downsampled) volume, in pixels */ - // If we ever allow downsampling in z, replace with Vector3 originalSize: Vector3; /** * XY dimensions of the texture atlas used by `RayMarchedAtlasVolume` and `Atlas2DSlice`, in number of z-slice @@ -41,6 +41,11 @@ export type ImageInfo = Readonly<{ /** Symbol of temporal unit used by `timeScale`, e.g. "hr" */ timeUnit: string; + /** Number of scale levels available for this volume */ + numMultiscaleLevels: number; + /** The scale level from which this image was loaded, between `0` and `numMultiscaleLevels-1` */ + multiscaleLevel: number; + transform: { /** Translation of the volume from the center of space, in volume voxels */ translation: Vector3; @@ -67,6 +72,8 @@ export const getDefaultImageInfo = (): ImageInfo => ({ times: 1, timeScale: 1, timeUnit: "", + numMultiscaleLevels: 1, + multiscaleLevel: 0, transform: { translation: new Vector3(0, 0, 0), rotation: new Vector3(0, 0, 0), @@ -229,14 +236,29 @@ export default class Volume { this.normRegionOffset = subregionOffset.clone().divide(volumeSize); } - updateRequiredData(required: Partial, onChannelLoaded?: PerChannelCallback) { + /** Call on any state update that may require new data to be loaded (subregion, enabled channels, time, etc.) */ + async updateRequiredData(required: Partial, onChannelLoaded?: PerChannelCallback): Promise { this.loadSpecRequired = { ...this.loadSpecRequired, ...required }; + let noReload = + this.loadSpec.time === this.loadSpecRequired.time && + this.loadSpec.subregion.containsBox(this.loadSpecRequired.subregion); + + // An update to `subregion` should trigger a reload when the new subregion is not contained in the old one + // OR when the new subregion is smaller than the old one by enough that we can load a higher scale level. + if (noReload && !this.loadSpec.subregion.equals(this.loadSpecRequired.subregion)) { + const currentScale = this.imageInfo.multiscaleLevel; + // `LoadSpec.multiscaleLevel`, if specified, forces a cap on the scale level we can load. + const minLevel = this.loadSpec.multiscaleLevel ?? 0; + // Loaders should cache loaded dimensions so that this call blocks no more than once per valid `LoadSpec`. + const dims = await this.loader?.loadDims(this.loadSpecRequired); + if (dims) { + const loadableLevel = estimateLevelForAtlas(dims.map(({ shape }) => [shape[2], shape[3], shape[4]])); + noReload = currentScale <= Math.max(loadableLevel, minLevel); + } + } + // if newly required data is not currently contained in this volume... - if ( - this.loadSpecRequired.time !== this.loadSpec.time || - this.loadSpecRequired.scene !== this.loadSpec.scene || - !this.loadSpec.subregion.containsBox(this.loadSpecRequired.subregion) - ) { + if (!noReload) { // ...clone `loadSpecRequired` into `loadSpec` and load this.setUnloaded(); this.loadSpec = { diff --git a/src/loaders/IVolumeLoader.ts b/src/loaders/IVolumeLoader.ts index 545bc751..48b3552c 100644 --- a/src/loaders/IVolumeLoader.ts +++ b/src/loaders/IVolumeLoader.ts @@ -3,7 +3,6 @@ import { Box3, Vector3 } from "three"; import Volume from "../Volume"; export class LoadSpec { - scene = 0; time = 0; multiscaleLevel?: number; /** Subregion of volume to load. If not specified, the entire volume is loaded. Specify as floats between 0-1. */ @@ -13,11 +12,10 @@ export class LoadSpec { export function loadSpecToString(spec: LoadSpec): string { const { min, max } = spec.subregion; - return `${spec.scene}:${spec.multiscaleLevel}:${spec.time}:x(${min.x},${max.x}):y(${min.y},${max.y}):z(${min.z},${max.z})`; + return `${spec.multiscaleLevel}:${spec.time}:x(${min.x},${max.x}):y(${min.y},${max.y}):z(${min.z},${max.z})`; } export class VolumeDims { - subpath = ""; // shape: [t, c, z, y, x] shape: number[] = [0, 0, 0, 0, 0]; // spacing: [t, c, z, y, x]; generally expect 1 for non-spatial dimensions diff --git a/src/loaders/JsonImageInfoLoader.ts b/src/loaders/JsonImageInfoLoader.ts index 2bb27253..b0b96d36 100644 --- a/src/loaders/JsonImageInfoLoader.ts +++ b/src/loaders/JsonImageInfoLoader.ts @@ -80,6 +80,9 @@ const convertImageInfo = (json: JsonImageInfo): ImageInfo => ({ timeScale: json.time_scale || 1, timeUnit: json.time_unit || "s", + numMultiscaleLevels: 1, + multiscaleLevel: 0, + transform: { translation: json.transform?.translation ? new Vector3().fromArray(json.transform.translation) @@ -92,8 +95,7 @@ const convertImageInfo = (json: JsonImageInfo): ImageInfo => ({ class JsonImageInfoLoader implements IVolumeLoader { urls: string[]; - time: number; - jsonInfo: JsonImageInfo | null = null; + jsonInfo: (JsonImageInfo | undefined)[]; cache?: VolumeCache; @@ -104,24 +106,29 @@ class JsonImageInfoLoader implements IVolumeLoader { this.urls = [urls]; } - this.time = 0; + this.jsonInfo = new Array(this.urls.length); this.cache = cache; } - async getJsonImageInfo(loadSpec: LoadSpec): Promise { - const response = await fetch(this.urls[loadSpec.time]); + private async getJsonImageInfo(time: number): Promise { + const cachedInfo = this.jsonInfo[time]; + if (cachedInfo) { + return cachedInfo; + } + + const response = await fetch(this.urls[time]); const imageInfo = (await response.json()) as JsonImageInfo; imageInfo.pixel_size_unit = imageInfo.pixel_size_unit || "μm"; imageInfo.times = imageInfo.times || this.urls.length; + this.jsonInfo[time] = imageInfo; return imageInfo; } async loadDims(loadSpec: LoadSpec): Promise { - const jsonInfo = await this.getJsonImageInfo(loadSpec); + const jsonInfo = await this.getJsonImageInfo(loadSpec.time); const d = new VolumeDims(); - d.subpath = ""; d.shape = [jsonInfo.times || 1, jsonInfo.channels, jsonInfo.tiles, jsonInfo.tile_height, jsonInfo.tile_width]; d.spacing = [1, 1, jsonInfo.pixel_size_z, jsonInfo.pixel_size_y, jsonInfo.pixel_size_x]; d.spaceUnit = jsonInfo.pixel_size_unit || "μm"; @@ -130,9 +137,8 @@ class JsonImageInfoLoader implements IVolumeLoader { } async createVolume(loadSpec: LoadSpec, onChannelLoaded?: PerChannelCallback): Promise { - this.time = loadSpec.time; - this.jsonInfo = await this.getJsonImageInfo(loadSpec); - const imageInfo = convertImageInfo(this.jsonInfo); + const jsonInfo = await this.getJsonImageInfo(loadSpec.time); + const imageInfo = convertImageInfo(jsonInfo); const vol = new Volume(imageInfo, loadSpec, this); vol.channelLoadCallback = onChannelLoaded; @@ -146,12 +152,9 @@ class JsonImageInfoLoader implements IVolumeLoader { // Try to figure out the urlPrefix from the LoadSpec. // For this format we assume the image data is in the same directory as the json file. const loadSpec = explicitLoadSpec || vol.loadSpec; - if (this.time !== loadSpec.time) { - this.time = loadSpec.time; - this.jsonInfo = await this.getJsonImageInfo(loadSpec); - } + const jsonInfo = await this.getJsonImageInfo(loadSpec.time); - let images = this.jsonInfo?.images; + let images = jsonInfo?.images; if (!images) { return; } @@ -163,7 +166,7 @@ class JsonImageInfoLoader implements IVolumeLoader { } // This regex removes everything after the last slash, so the url had better be simple. - const urlPrefix = this.urls[this.time].replace(/[^/]*$/, ""); + const urlPrefix = this.urls[loadSpec.time].replace(/[^/]*$/, ""); images = images.map((element) => ({ ...element, name: urlPrefix + element.name })); vol.loadSpec = { diff --git a/src/loaders/OmeZarrLoader.ts b/src/loaders/OmeZarrLoader.ts index 9f1468a4..e2da592b 100644 --- a/src/loaders/OmeZarrLoader.ts +++ b/src/loaders/OmeZarrLoader.ts @@ -17,7 +17,6 @@ import { unitNameToSymbol, } from "./VolumeLoaderUtils"; -const MAX_ATLAS_DIMENSION = 2048; const CHUNK_REQUEST_CANCEL_REASON = "chunk request cancelled"; type CoordinateTransformation = @@ -194,22 +193,21 @@ function remapAxesToTCZYX(axes: Axis[]): [number, number, number, number, number return axisTCZYX; } -function pickLevelToLoad(loadSpec: LoadSpec, zarrMultiscales: ZarrArray[], zi: number, yi: number, xi: number): number { +/** + * Picks the best scale level to load based on scale level dimensions, a max atlas size, and a `LoadSpec`. + * This works like `estimateLevelForAtlas` but factors in `LoadSpec`'s `subregion` property (shrinks the size of the + * data, maybe enough to allow loading a higher level) and its `multiscaleLevel` property (sets a max scale level). + */ +function pickLevelToLoad(loadSpec: LoadSpec, spatialDimsZYX: [number, number, number][]): number { const size = loadSpec.subregion.getSize(new Vector3()); - - const spatialDims = zarrMultiscales.map(({ shape }) => [ - Math.max(shape[zi] * size.z, 1), - Math.max(shape[yi] * size.y, 1), - Math.max(shape[xi] * size.x, 1), + const dims = spatialDimsZYX.map(([z, y, x]): [number, number, number] => [ + Math.max(z * size.z, 1), + Math.max(y * size.y, 1), + Math.max(x * size.x, 1), ]); - const optimalLevel = estimateLevelForAtlas(spatialDims, MAX_ATLAS_DIMENSION); - - if (loadSpec.multiscaleLevel) { - return Math.max(optimalLevel, loadSpec.multiscaleLevel); - } else { - return optimalLevel; - } + const optimalLevel = estimateLevelForAtlas(dims); + return Math.max(optimalLevel, loadSpec.multiscaleLevel ?? 0); } function convertChannel(channelData: TypedArray, dtype: string): Uint8Array { @@ -338,7 +336,12 @@ class OMEZarrLoader implements IVolumeLoader { return scale; } - async loadDims(loadSpec: LoadSpec): Promise { + private getLevelShapesZYX(): [number, number, number][] { + const [z, y, x] = this.axesTCZYX.slice(-3); + return this.scaleLevels.map(({ shape }) => [shape[z], shape[y], shape[x]]); + } + + loadDims(loadSpec: LoadSpec): Promise { const [spaceUnit, timeUnit] = this.getUnitSymbols(); // Compute subregion size so we can factor that in const maxExtent = this.maxExtent ?? new Box3(new Vector3(0, 0, 0), new Vector3(1, 1, 1)); @@ -352,7 +355,6 @@ class OMEZarrLoader implements IVolumeLoader { dims.spaceUnit = spaceUnit; dims.timeUnit = timeUnit; - dims.subpath = level.path ?? ""; dims.shape = [-1, -1, -1, -1, -1]; dims.spacing = [1, 1, 1, 1, 1]; @@ -366,7 +368,7 @@ class OMEZarrLoader implements IVolumeLoader { return dims; }); - return result; + return Promise.resolve(result); } async createVolume(loadSpec: LoadSpec, onChannelLoaded?: PerChannelCallback): Promise { @@ -375,7 +377,7 @@ class OMEZarrLoader implements IVolumeLoader { const hasC = c > -1; const shape0 = this.scaleLevels[0].shape; - const levelToLoad = pickLevelToLoad(loadSpec, this.scaleLevels, z, y, x); + const levelToLoad = pickLevelToLoad(loadSpec, this.getLevelShapesZYX()); const shapeLv = this.scaleLevels[levelToLoad].shape; const [spatialUnit, timeUnit] = this.getUnitSymbols(); @@ -413,6 +415,8 @@ class OMEZarrLoader implements IVolumeLoader { times, timeScale, timeUnit, + numMultiscaleLevels: this.scaleLevels.length, + multiscaleLevel: levelToLoad, transform: { translation: new Vector3(0, 0, 0), @@ -442,7 +446,7 @@ class OMEZarrLoader implements IVolumeLoader { const [z, y, x] = this.axesTCZYX.slice(2); const subregion = composeSubregion(vol.loadSpec.subregion, maxExtent); - const levelIdx = pickLevelToLoad({ ...vol.loadSpec, subregion }, this.scaleLevels, z, y, x); + const levelIdx = pickLevelToLoad({ ...vol.loadSpec, subregion }, this.getLevelShapesZYX()); const level = this.scaleLevels[levelIdx]; const levelShape = level.shape; @@ -458,6 +462,7 @@ class OMEZarrLoader implements IVolumeLoader { volumeSize: volSizePx, subregionSize: regionSizePx, subregionOffset: regionPx.min, + multiscaleLevel: levelIdx, }; vol.updateDimensions(); diff --git a/src/loaders/OpenCellLoader.ts b/src/loaders/OpenCellLoader.ts index 42cda5ad..6f974acd 100644 --- a/src/loaders/OpenCellLoader.ts +++ b/src/loaders/OpenCellLoader.ts @@ -9,7 +9,6 @@ import { JsonImageInfoLoader } from "./JsonImageInfoLoader"; class OpenCellLoader implements IVolumeLoader { async loadDims(_: LoadSpec): Promise { const d = new VolumeDims(); - d.subpath = ""; d.shape = [1, 2, 27, 600, 600]; d.spacing = [1, 1, 2, 1, 1]; d.spaceUnit = ""; // unknown unit. @@ -41,6 +40,9 @@ class OpenCellLoader implements IVolumeLoader { timeScale: 1, timeUnit: "", + numMultiscaleLevels: 1, + multiscaleLevel: 0, + transform: { translation: new Vector3(0, 0, 0), rotation: new Vector3(0, 0, 0), diff --git a/src/loaders/TiffLoader.ts b/src/loaders/TiffLoader.ts index 4890282d..a7d099e3 100644 --- a/src/loaders/TiffLoader.ts +++ b/src/loaders/TiffLoader.ts @@ -60,43 +60,35 @@ function getOMEDims(imageEl: Element): OMEDims { return dims; } -async function getDimsFromUrl(url: string): Promise { - const tiff = await fromUrl(url, { allowFullFile: true }); - // DO NOT DO THIS, ITS SLOW - // const imagecount = await tiff.getImageCount(); - // read the FIRST image - const image = await tiff.getImage(); - - const tiffimgdesc = prepareXML(image.getFileDirectory().ImageDescription); - const omeEl = getOME(tiffimgdesc); - - const image0El = omeEl.getElementsByTagName("Image")[0]; - return getOMEDims(image0El); -} - const getBytesPerSample = (type: string): number => (type === "uint8" ? 1 : type === "uint16" ? 2 : 4); class TiffLoader implements IVolumeLoader { url: string; - dimensionOrder?: string; - bytesPerSample?: number; + dims?: OMEDims; constructor(url: string) { this.url = url; } - async loadDims(_loadSpec: LoadSpec): Promise { - const tiff = await fromUrl(this.url, { allowFullFile: true }); - // DO NOT DO THIS, ITS SLOW - // const imagecount = await tiff.getImageCount(); - // read the FIRST image - const image = await tiff.getImage(); + private async loadOmeDims(): Promise { + if (!this.dims) { + const tiff = await fromUrl(this.url, { allowFullFile: true }); + // DO NOT DO THIS, ITS SLOW + // const imagecount = await tiff.getImageCount(); + // read the FIRST image + const image = await tiff.getImage(); - const tiffimgdesc = prepareXML(image.getFileDirectory().ImageDescription); - const omeEl = getOME(tiffimgdesc); + const tiffimgdesc = prepareXML(image.getFileDirectory().ImageDescription); + const omeEl = getOME(tiffimgdesc); - const image0El = omeEl.getElementsByTagName("Image")[0]; - const dims = getOMEDims(image0El); + const image0El = omeEl.getElementsByTagName("Image")[0]; + this.dims = getOMEDims(image0El); + } + return this.dims; + } + + async loadDims(_loadSpec: LoadSpec): Promise { + const dims = await this.loadOmeDims(); const d = new VolumeDims(); d.shape = [dims.sizet, dims.sizec, dims.sizez, dims.sizey, dims.sizex]; @@ -107,7 +99,7 @@ class TiffLoader implements IVolumeLoader { } async createVolume(_loadSpec: LoadSpec, onChannelLoaded?: PerChannelCallback): Promise { - const dims = await getDimsFromUrl(this.url); + const dims = await this.loadOmeDims(); // compare with sizex, sizey //const width = image.getWidth(); //const height = image.getHeight(); @@ -140,6 +132,9 @@ class TiffLoader implements IVolumeLoader { timeScale: 1, timeUnit: "", + numMultiscaleLevels: 1, + multiscaleLevel: 0, + transform: { translation: new Vector3(0, 0, 0), rotation: new Vector3(0, 0, 0), @@ -151,20 +146,11 @@ class TiffLoader implements IVolumeLoader { vol.channelLoadCallback = onChannelLoaded; vol.imageMetadata = buildDefaultMetadata(imgdata); - this.dimensionOrder = dims.dimensionorder; - this.bytesPerSample = getBytesPerSample(dims.pixeltype); - return vol; } - async loadVolumeData(vol: Volume, loadSpec?: LoadSpec, onChannelLoaded?: PerChannelCallback): Promise { - if (this.bytesPerSample === undefined || this.dimensionOrder === undefined) { - const dims = await getDimsFromUrl(this.url); - - this.dimensionOrder = dims.dimensionorder; - this.bytesPerSample = getBytesPerSample(dims.pixeltype); - } - + async loadVolumeData(vol: Volume, _loadSpec?: LoadSpec, onChannelLoaded?: PerChannelCallback): Promise { + const dims = await this.loadOmeDims(); const imageInfo = vol.imageInfo; // do each channel on a worker? @@ -177,8 +163,8 @@ class TiffLoader implements IVolumeLoader { tilesizey: imageInfo.volumeSize.y, sizec: imageInfo.numChannels, sizez: imageInfo.volumeSize.z, - dimensionOrder: this.dimensionOrder, - bytesPerSample: this.bytesPerSample, + dimensionOrder: dims.dimensionorder, + bytesPerSample: getBytesPerSample(dims.pixeltype), url: this.url, }; const worker = new Worker(new URL("../workers/FetchTiffWorker", import.meta.url)); @@ -195,9 +181,6 @@ class TiffLoader implements IVolumeLoader { }; worker.postMessage(params); } - - this.dimensionOrder = undefined; - this.bytesPerSample = undefined; } } diff --git a/src/loaders/VolumeLoaderUtils.ts b/src/loaders/VolumeLoaderUtils.ts index d22f5d14..217b42ff 100644 --- a/src/loaders/VolumeLoaderUtils.ts +++ b/src/loaders/VolumeLoaderUtils.ts @@ -3,6 +3,8 @@ import { Box3, Vector2, Vector3 } from "three"; import { ImageInfo } from "../Volume"; +const MAX_ATLAS_EDGE = 2048; + // Map from units to their symbols const UNIT_SYMBOLS = { angstrom: "Å", @@ -74,9 +76,13 @@ export function computePackedAtlasDims(z: number, tw: number, th: number): Vecto return new Vector2(nrows, ncols); } -export function estimateLevelForAtlas(spatialDimsZYX: number[][], maxAtlasEdge = 4096) { - // update levelToLoad after we get size info about multiscales. - // decide to max out at a 4k x 4k texture. +/** Picks the largest scale level that can fit into a texture atlas */ +export function estimateLevelForAtlas(spatialDimsZYX: [number, number, number][], maxAtlasEdge = MAX_ATLAS_EDGE) { + if (spatialDimsZYX.length <= 1) { + return 0; + } + + // update levelToLoad after we get size info about multiscales let levelToLoad = spatialDimsZYX.length - 1; for (let i = 0; i < spatialDimsZYX.length; ++i) { // estimate atlas size: diff --git a/src/test/volume.test.ts b/src/test/volume.test.ts index c6f9c920..01ff6665 100644 --- a/src/test/volume.test.ts +++ b/src/test/volume.test.ts @@ -35,6 +35,9 @@ const testimgdata: ImageInfo = { timeScale: 1, timeUnit: "", + numMultiscaleLevels: 1, + multiscaleLevel: 0, + transform: { translation: new Vector3(0, 0, 0), rotation: new Vector3(0, 0, 0),