diff --git a/examples/client/index.html b/examples/client/index.html
index b14cbed2..b83c1159 100644
--- a/examples/client/index.html
+++ b/examples/client/index.html
@@ -90,6 +90,7 @@
Results
Appendable.init(
"green_tripdata_2023-01.jsonl",
"green_tripdata_2023-01.index",
+ { useMultipartByteRanges: false },
).then(async (db) => {
// populate fields
db.fields().then((fields) => {
diff --git a/src/data-file.ts b/src/data-file.ts
index a6398040..eac00772 100644
--- a/src/data-file.ts
+++ b/src/data-file.ts
@@ -1,14 +1,15 @@
+import { Config } from ".";
import { requestRanges } from "./range-request";
-import { LengthIntegrityError, RangeResolver } from "./resolver";
+import { RangeResolver } from "./resolver";
export class DataFile {
private originalResolver?: RangeResolver;
private constructor(private resolver: RangeResolver) {}
- static forUrl(url: string) {
+ static forUrl(url: string, config: Config) {
return DataFile.forResolver(
- async (ranges) => await requestRanges(url, ranges),
+ async (ranges) => await requestRanges(url, ranges, config),
);
}
diff --git a/src/index-file/index-file.ts b/src/index-file/index-file.ts
index 08b5c67f..aaa8c00f 100644
--- a/src/index-file/index-file.ts
+++ b/src/index-file/index-file.ts
@@ -13,11 +13,12 @@ import {
} from "./meta";
import { FieldType } from "../db/database";
import { requestRanges } from "../range-request";
+import { Config } from "..";
export class IndexFile {
- static async forUrl(url: string) {
+ static async forUrl(url: string, config: Config) {
return await IndexFile.forResolver(
- async (ranges) => await requestRanges(url, ranges),
+ async (ranges) => await requestRanges(url, ranges, config),
);
}
diff --git a/src/index.ts b/src/index.ts
index 7b11251c..479d0eb8 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -3,16 +3,21 @@ import { Database, FieldType, fieldTypeToString } from "./db/database";
import { IndexFile } from "./index-file/index-file";
import { RangeResolver } from "./resolver";
+export type Config = {
+ useMultipartByteRanges?: boolean;
+};
+
export async function init(
dataUrl: string | RangeResolver,
indexUrl: string | RangeResolver,
+ config: Config,
) {
return Database.forDataFileAndIndexFile(
typeof dataUrl === "string"
- ? DataFile.forUrl(dataUrl)
+ ? DataFile.forUrl(dataUrl, config)
: DataFile.forResolver(dataUrl),
typeof indexUrl === "string"
- ? await IndexFile.forUrl(indexUrl)
+ ? await IndexFile.forUrl(indexUrl, config)
: await IndexFile.forResolver(indexUrl),
);
}
diff --git a/src/range-request.ts b/src/range-request.ts
index f4dc6330..7eed9305 100644
--- a/src/range-request.ts
+++ b/src/range-request.ts
@@ -1,10 +1,48 @@
+import { Config } from ".";
import parseMultipartBody from "./multipart";
import { LengthIntegrityError } from "./resolver";
+async function resolveIndividualPromises(
+ url: string,
+ ranges: { start: number; end: number; expectedLength?: number }[],
+) {
+ console.log("resolving ranges individually");
+ // fallback to resolving ranges individually
+ const individualRangePromises = ranges.map(
+ async ({ start, end, expectedLength }) => {
+ const rangeHeader = `${start}-${end}`;
+ const res = await fetch(url, {
+ headers: { Range: `bytes=${rangeHeader}` },
+ });
+
+ const totalLength = Number(
+ res.headers.get("Content-Range")!.split("/")[1],
+ );
+ if (expectedLength && totalLength !== expectedLength) {
+ throw new LengthIntegrityError();
+ }
+ return {
+ data: await res.arrayBuffer(),
+ totalLength: totalLength,
+ };
+ },
+ );
+ return await Promise.all(individualRangePromises);
+}
+
export async function requestRanges(
url: string,
ranges: { start: number; end: number; expectedLength?: number }[],
+ config: Config,
): Promise<{ data: ArrayBuffer; totalLength: number }[]> {
+ const { useMultipartByteRanges } = config;
+ if (
+ useMultipartByteRanges === false ||
+ useMultipartByteRanges === undefined
+ ) {
+ return await resolveIndividualPromises(url, ranges);
+ }
+
const rangesHeader = ranges
.map(({ start, end }) => `${start}-${end}`)
.join(",");
@@ -18,28 +56,10 @@ export async function requestRanges(
switch (response.status) {
case 200:
- // fallback to resolving ranges individually
- const individualRangePromises = ranges.map(
- async ({ start, end, expectedLength }) => {
- const rangeHeader = `${start}-${end}`;
- const res = await fetch(url, {
- headers: { Range: `bytes=${rangeHeader}` },
- });
-
- const totalLength = Number(
- res.headers.get("Content-Range")!.split("/")[1],
- );
- if (expectedLength && totalLength !== expectedLength) {
- throw new LengthIntegrityError();
- }
- return {
- data: await res.arrayBuffer(),
- totalLength: totalLength,
- };
- },
+ console.warn(
+ `useMultipartByteRanges has not been set to false. The server can not handle multipart byte ranges.`,
);
- return await Promise.all(individualRangePromises);
-
+ return await resolveIndividualPromises(url, ranges);
case 206:
const contentType = response.headers.get("Content-Type");
if (!contentType) {