Merge pull request #131 from nomic-ai/08-07-import_quadfeather_manifests

import quadfeather manifests

src/Deeptable.ts

@@ -1139,3 +1139,77 @@ function maxTwo(a: number, b: number) {
 function minTwo(a: number, b: number) {
   return a > b ? b : a;
 }
+
+type RowFormatManifest = {
+  key: string;
+  nPoints: number;
+  min_ix: number;
+  max_ix: number;
+  extent: string; // JSON deserializes to a rectangle.
+};
+
+export type TileManifest = {
+  key: string;
+  // The number of data points in that specific tile.
+  nPoints: number;
+  children: TileManifest[];
+  min_ix: number;
+  max_ix: number;
+  extent: Rectangle;
+};
+
+export async function tileManifest(url: string) {
+  const data = await fetch(url).then((d) => d.arrayBuffer());
+  const tb = tableFromIPC(data);
+  const rows: RowFormatManifest[] = [...tb].map(
+    ({ key, nPoints, min_ix, max_ix, extent }: RowFormatManifest) => {
+      return {
+        key, // string
+        nPoints: Number(nPoints), // Number because this can come in as a BigInt
+        min_ix: Number(min_ix), // BigInt -> Number cast
+        max_ix: Number(max_ix), // BigInt -> Number cast.
+        extent, // Leave as unparsed json.
+      } as const;
+    },
+  );
+  return parseManifest(rows);
+}
+
+/**
+ * We receive a manifest as a list of tiles, but deepscatter wants to receive a nested tree.
+ * This function converts from one to the other, using the format of the quadtree to check
+ * which of all possible children exist.
+ *
+ * @param raw The manifest as it comes in from the
+ * @returns A tree-structured manifest suitable for passing to deeptable's "tileManifest" argument.
+ */
+export function parseManifest(rows: RowFormatManifest[]): TileManifest {
+  const lookup = Object.fromEntries(rows.map((d) => [d.key, d]));
+
+  /**
+   * Given a row, finds its children and recursively
+   * @param input The row to insert.
+   * @returns A tree-shaped manifest.
+   */
+  function buildNetwork(input: RowFormatManifest): TileManifest {
+    const children: TileManifest[] = [];
+    const [z, x, y] = input.key.split('/').map((i) => parseInt(i));
+    for (const i of [0, 1]) {
+      for (const j of [0, 1]) {
+        const childKey = `${z + 1}/${x * 2 + i}/${y * 2 + j}`;
+        if (lookup[childKey]) {
+          const manifest = buildNetwork(lookup[childKey]);
+          children.push(manifest);
+        }
+      }
+    }
+    return {
+      ...input,
+      extent: JSON.parse(input.extent) as Rectangle,
+      children,
+    };
+  }
+  const networked = buildNetwork(lookup['0/0/0']);
+
+  return networked;
+}