diff --git a/examples/client/index.html b/examples/client/index.html index b14cbed2..b83c1159 100644 --- a/examples/client/index.html +++ b/examples/client/index.html @@ -90,6 +90,7 @@

Results

Appendable.init( "green_tripdata_2023-01.jsonl", "green_tripdata_2023-01.index", + { useMultipartByteRanges: false }, ).then(async (db) => { // populate fields db.fields().then((fields) => { diff --git a/src/data-file.ts b/src/data-file.ts index a6398040..eac00772 100644 --- a/src/data-file.ts +++ b/src/data-file.ts @@ -1,14 +1,15 @@ +import { Config } from "."; import { requestRanges } from "./range-request"; -import { LengthIntegrityError, RangeResolver } from "./resolver"; +import { RangeResolver } from "./resolver"; export class DataFile { private originalResolver?: RangeResolver; private constructor(private resolver: RangeResolver) {} - static forUrl(url: string) { + static forUrl(url: string, config: Config) { return DataFile.forResolver( - async (ranges) => await requestRanges(url, ranges), + async (ranges) => await requestRanges(url, ranges, config), ); } diff --git a/src/index-file/index-file.ts b/src/index-file/index-file.ts index 08b5c67f..aaa8c00f 100644 --- a/src/index-file/index-file.ts +++ b/src/index-file/index-file.ts @@ -13,11 +13,12 @@ import { } from "./meta"; import { FieldType } from "../db/database"; import { requestRanges } from "../range-request"; +import { Config } from ".."; export class IndexFile { - static async forUrl(url: string) { + static async forUrl(url: string, config: Config) { return await IndexFile.forResolver( - async (ranges) => await requestRanges(url, ranges), + async (ranges) => await requestRanges(url, ranges, config), ); } diff --git a/src/index.ts b/src/index.ts index 7b11251c..479d0eb8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,16 +3,21 @@ import { Database, FieldType, fieldTypeToString } from "./db/database"; import { IndexFile } from "./index-file/index-file"; import { RangeResolver } from "./resolver"; +export type Config = { + useMultipartByteRanges?: boolean; +}; + export async function init( dataUrl: string | RangeResolver, indexUrl: string | RangeResolver, + config: Config, ) { return Database.forDataFileAndIndexFile( typeof dataUrl === "string" - ? DataFile.forUrl(dataUrl) + ? DataFile.forUrl(dataUrl, config) : DataFile.forResolver(dataUrl), typeof indexUrl === "string" - ? await IndexFile.forUrl(indexUrl) + ? await IndexFile.forUrl(indexUrl, config) : await IndexFile.forResolver(indexUrl), ); } diff --git a/src/range-request.ts b/src/range-request.ts index f4dc6330..7eed9305 100644 --- a/src/range-request.ts +++ b/src/range-request.ts @@ -1,10 +1,48 @@ +import { Config } from "."; import parseMultipartBody from "./multipart"; import { LengthIntegrityError } from "./resolver"; +async function resolveIndividualPromises( + url: string, + ranges: { start: number; end: number; expectedLength?: number }[], +) { + console.log("resolving ranges individually"); + // fallback to resolving ranges individually + const individualRangePromises = ranges.map( + async ({ start, end, expectedLength }) => { + const rangeHeader = `${start}-${end}`; + const res = await fetch(url, { + headers: { Range: `bytes=${rangeHeader}` }, + }); + + const totalLength = Number( + res.headers.get("Content-Range")!.split("/")[1], + ); + if (expectedLength && totalLength !== expectedLength) { + throw new LengthIntegrityError(); + } + return { + data: await res.arrayBuffer(), + totalLength: totalLength, + }; + }, + ); + return await Promise.all(individualRangePromises); +} + export async function requestRanges( url: string, ranges: { start: number; end: number; expectedLength?: number }[], + config: Config, ): Promise<{ data: ArrayBuffer; totalLength: number }[]> { + const { useMultipartByteRanges } = config; + if ( + useMultipartByteRanges === false || + useMultipartByteRanges === undefined + ) { + return await resolveIndividualPromises(url, ranges); + } + const rangesHeader = ranges .map(({ start, end }) => `${start}-${end}`) .join(","); @@ -18,28 +56,10 @@ export async function requestRanges( switch (response.status) { case 200: - // fallback to resolving ranges individually - const individualRangePromises = ranges.map( - async ({ start, end, expectedLength }) => { - const rangeHeader = `${start}-${end}`; - const res = await fetch(url, { - headers: { Range: `bytes=${rangeHeader}` }, - }); - - const totalLength = Number( - res.headers.get("Content-Range")!.split("/")[1], - ); - if (expectedLength && totalLength !== expectedLength) { - throw new LengthIntegrityError(); - } - return { - data: await res.arrayBuffer(), - totalLength: totalLength, - }; - }, + console.warn( + `useMultipartByteRanges has not been set to false. The server can not handle multipart byte ranges.`, ); - return await Promise.all(individualRangePromises); - + return await resolveIndividualPromises(url, ranges); case 206: const contentType = response.headers.get("Content-Type"); if (!contentType) {