feat: add useMultipart flag

kevmo314 · Mar 4, 2024 · 3dfe48f · 3dfe48f
1 parent 9bf2ee5
commit 3dfe48f
Show file tree

Hide file tree

Showing 5 changed files with 56 additions and 28 deletions.
diff --git a/examples/client/index.html b/examples/client/index.html
@@ -90,6 +90,7 @@ <h2>Results</h2>
         Appendable.init(
           "green_tripdata_2023-01.jsonl",
           "green_tripdata_2023-01.index",
+          { useMultipartByteRanges: false },
         ).then(async (db) => {
           // populate fields
           db.fields().then((fields) => {

diff --git a/src/data-file.ts b/src/data-file.ts
@@ -1,14 +1,15 @@
+import { Config } from ".";
 import { requestRanges } from "./range-request";
-import { LengthIntegrityError, RangeResolver } from "./resolver";
+import { RangeResolver } from "./resolver";
 
 export class DataFile {
   private originalResolver?: RangeResolver;
 
   private constructor(private resolver: RangeResolver) {}
 
-  static forUrl(url: string) {
+  static forUrl(url: string, config: Config) {
     return DataFile.forResolver(
-      async (ranges) => await requestRanges(url, ranges),
+      async (ranges) => await requestRanges(url, ranges, config),
     );
   }
 

diff --git a/src/index-file/index-file.ts b/src/index-file/index-file.ts
@@ -13,11 +13,12 @@ import {
 } from "./meta";
 import { FieldType } from "../db/database";
 import { requestRanges } from "../range-request";
+import { Config } from "..";
 
 export class IndexFile {
-  static async forUrl<T = any>(url: string) {
+  static async forUrl<T = any>(url: string, config: Config) {
     return await IndexFile.forResolver<T>(
-      async (ranges) => await requestRanges(url, ranges),
+      async (ranges) => await requestRanges(url, ranges, config),
     );
   }
 

diff --git a/src/index.ts b/src/index.ts
@@ -3,16 +3,21 @@ import { Database, FieldType, fieldTypeToString } from "./db/database";
 import { IndexFile } from "./index-file/index-file";
 import { RangeResolver } from "./resolver";
 
+export type Config = {
+  useMultipartByteRanges?: boolean;
+};
+
 export async function init(
   dataUrl: string | RangeResolver,
   indexUrl: string | RangeResolver,
+  config: Config,
 ) {
   return Database.forDataFileAndIndexFile(
     typeof dataUrl === "string"
-      ? DataFile.forUrl(dataUrl)
+      ? DataFile.forUrl(dataUrl, config)
       : DataFile.forResolver(dataUrl),
     typeof indexUrl === "string"
-      ? await IndexFile.forUrl(indexUrl)
+      ? await IndexFile.forUrl(indexUrl, config)
       : await IndexFile.forResolver(indexUrl),
   );
 }

diff --git a/src/range-request.ts b/src/range-request.ts
@@ -1,10 +1,48 @@
+import { Config } from ".";
 import parseMultipartBody from "./multipart";
 import { LengthIntegrityError } from "./resolver";
 
+async function resolveIndividualPromises(
+  url: string,
+  ranges: { start: number; end: number; expectedLength?: number }[],
+) {
+  console.log("resolving ranges individually");
+  // fallback to resolving ranges individually
+  const individualRangePromises = ranges.map(
+    async ({ start, end, expectedLength }) => {
+      const rangeHeader = `${start}-${end}`;
+      const res = await fetch(url, {
+        headers: { Range: `bytes=${rangeHeader}` },
+      });
+
+      const totalLength = Number(
+        res.headers.get("Content-Range")!.split("/")[1],
+      );
+      if (expectedLength && totalLength !== expectedLength) {
+        throw new LengthIntegrityError();
+      }
+      return {
+        data: await res.arrayBuffer(),
+        totalLength: totalLength,
+      };
+    },
+  );
+  return await Promise.all(individualRangePromises);
+}
+
 export async function requestRanges(
   url: string,
   ranges: { start: number; end: number; expectedLength?: number }[],
+  config: Config,
 ): Promise<{ data: ArrayBuffer; totalLength: number }[]> {
+  const { useMultipartByteRanges } = config;
+  if (
+    useMultipartByteRanges === false ||
+    useMultipartByteRanges === undefined
+  ) {
+    return await resolveIndividualPromises(url, ranges);
+  }
+
   const rangesHeader = ranges
     .map(({ start, end }) => `${start}-${end}`)
     .join(",");
@@ -18,28 +56,10 @@ export async function requestRanges(
 
   switch (response.status) {
     case 200:
-      // fallback to resolving ranges individually
-      const individualRangePromises = ranges.map(
-        async ({ start, end, expectedLength }) => {
-          const rangeHeader = `${start}-${end}`;
-          const res = await fetch(url, {
-            headers: { Range: `bytes=${rangeHeader}` },
-          });
-
-          const totalLength = Number(
-            res.headers.get("Content-Range")!.split("/")[1],
-          );
-          if (expectedLength && totalLength !== expectedLength) {
-            throw new LengthIntegrityError();
-          }
-          return {
-            data: await res.arrayBuffer(),
-            totalLength: totalLength,
-          };
-        },
+      console.warn(
+        `useMultipartByteRanges has not been set to false. The server can not handle multipart byte ranges.`,
       );
-      return await Promise.all(individualRangePromises);
-
+      return await resolveIndividualPromises(url, ranges);
     case 206:
       const contentType = response.headers.get("Content-Type");
       if (!contentType) {