Remove the offset-based pagination

front/components/spaces/SpaceDataSourceViewContentList.tsx

@@ -1,4 +1,4 @@
-import type { MenuItem } from "@dust-tt/sparkle";
+import type { HistoryOptions, MenuItem } from "@dust-tt/sparkle";
 import {
   Button,
   cn,
@@ -11,7 +11,6 @@ import {
   SearchInput,
   Spinner,
   useHashParam,
-  usePaginationFromUrl,
   useSendNotification,
 } from "@dust-tt/sparkle";
 import type {
@@ -32,6 +31,7 @@ import {
 import type {
   CellContext,
   ColumnDef,
+  PaginationState,
   SortingState,
 } from "@tanstack/react-table";
 import { useRouter } from "next/router";
@@ -175,8 +175,8 @@ function useStaticDataSourceViewHasContent({
       owner,
       internalIds: parentId ? [parentId] : undefined,
       pagination: {
-        pageIndex: 0,
-        pageSize: 1,
+        cursor: null,
+        limit: 1,
       },
       viewType,
     });
@@ -227,10 +227,16 @@ export const SpaceDataSourceViewContentList = ({
   const [sorting, setSorting] = React.useState<SortingState>([]);
   const contentActionsRef = useRef<ContentActionsRef>(null);
 
-  const { pagination, setPagination } = usePaginationFromUrl({
+  // State for cursor pagination (in URL).
+  const { cursorPagination, setCursorPagination } = useCursorPaginationFromUrl({
     urlPrefix: "table",
     initialPageSize: 25,
   });
+  // State for DataTable pagination.
+  const [tablePagination, setTablePagination] = useState<PaginationState>({
+    pageIndex: cursorPagination.cursor ? 1 : 0,
+    pageSize: cursorPagination.pageSize,
+  });
   const [viewType, setViewType] = useHashParam(
     "viewType",
     DEFAULT_VIEW_TYPE
@@ -248,14 +254,14 @@ export const SpaceDataSourceViewContentList = ({
   const handleViewTypeChange = useCallback(
     (newViewType: ContentNodesViewType) => {
       if (newViewType !== viewType) {
-        setPagination(
-          { pageIndex: 0, pageSize: pagination.pageSize },
+        setCursorPagination(
+          { cursor: null, pageSize: cursorPagination.pageSize },
           "replace"
         );
         setViewType(newViewType);
       }
     },
-    [setPagination, setViewType, viewType, pagination.pageSize]
+    [setCursorPagination, setViewType, viewType, cursorPagination.pageSize]
   );
 
   const { searchResultNodes, isSearchLoading, isSearchValidating } =
@@ -266,10 +272,6 @@ export const SpaceDataSourceViewContentList = ({
       search: debouncedSearch,
     });
 
-  // TODO(20250127, nodes-core): turn to true and remove when implementing pagination
-  const isServerPagination = false;
-  // isFolder(dataSourceView.dataSource) && !dataSourceSearch;
-
   const columns = useMemo(
     () => getTableColumns(showSpaceUsage),
     [showSpaceUsage]
@@ -279,12 +281,16 @@ export const SpaceDataSourceViewContentList = ({
     isNodesLoading,
     mutateRegardlessOfQueryParams: mutateContentNodes,
     nodes: childrenNodes,
+    nextPageCursor,
     totalNodesCount,
   } = useDataSourceViewContentNodes({
     dataSourceView,
     owner,
     parentId,
-    pagination: isServerPagination ? pagination : undefined,
+    pagination: {
+      cursor: cursorPagination.cursor,
+      limit: cursorPagination.pageSize,
+    },
     viewType: isValidContentNodesViewType(viewType)
       ? viewType
       : DEFAULT_VIEW_TYPE,
@@ -325,6 +331,37 @@ export const SpaceDataSourceViewContentList = ({
       viewType: "tables",
     });
 
+  const handlePaginationChange = useCallback(
+    (newTablePagination: PaginationState) => {
+      if (newTablePagination.pageSize !== tablePagination.pageSize) {
+        // Handle page size change
+        setTablePagination(newTablePagination);
+        setCursorPagination({
+          cursor: null,
+          pageSize: newTablePagination.pageSize,
+        });
+      } else if (
+        newTablePagination.pageIndex > tablePagination.pageIndex &&
+        nextPageCursor
+      ) {
+        // Next page - use cursor
+        setTablePagination(newTablePagination);
+        setCursorPagination({
+          cursor: nextPageCursor,
+          pageSize: tablePagination.pageSize,
+        });
+      } else if (newTablePagination.pageIndex < tablePagination.pageIndex) {
+        // Previous page - reset cursor
+        setTablePagination(newTablePagination);
+        setCursorPagination({
+          cursor: null,
+          pageSize: tablePagination.pageSize,
+        });
+      }
+    },
+    [tablePagination, nextPageCursor, setCursorPagination]
+  );
+
   const isDataSourceManaged = isManaged(dataSourceView.dataSource);
 
   const addToSpace = useCallback(
@@ -569,8 +606,8 @@ export const SpaceDataSourceViewContentList = ({
                 placeholder="Search (Name)"
                 value={dataSourceSearch}
                 onChange={(s) => {
-                  setPagination(
-                    { pageIndex: 0, pageSize: pagination.pageSize },
+                  setCursorPagination(
+                    { cursor: null, pageSize: cursorPagination.pageSize },
                     "replace"
                   );
                   setDataSourceSearch(s);
@@ -684,10 +721,10 @@ export const SpaceDataSourceViewContentList = ({
             )}
             sorting={sorting}
             setSorting={setSorting}
-            totalRowCount={isServerPagination ? totalNodesCount : undefined}
+            totalRowCount={totalNodesCount}
             rowCountIsCapped={totalNodesCount === ROWS_COUNT_CAPPED}
-            pagination={pagination}
-            setPagination={setPagination}
+            pagination={tablePagination}
+            setPagination={handlePaginationChange}
             columnsBreakpoints={columnsBreakpoints}
           />
         )}

front/lib/api/data_source_view.ts

@@ -14,11 +14,7 @@ import {
   FOLDERS_TO_HIDE_IF_EMPTY_MIME_TYPES,
   getContentNodeFromCoreNode,
 } from "@app/lib/api/content_nodes";
-import type {
-  CursorPaginationParams,
-  OffsetPaginationParams,
-} from "@app/lib/api/pagination";
-import { isCursorPaginationParams } from "@app/lib/api/pagination";
+import type { CursorPaginationParams } from "@app/lib/api/pagination";
 import type { Authenticator } from "@app/lib/auth";
 import type { DustError } from "@app/lib/error";
 import { DataSourceViewResource } from "@app/lib/resources/data_source_view_resource";
@@ -70,14 +66,14 @@ export function filterAndCropContentNodesByView(
 interface GetContentNodesForDataSourceViewParams {
   internalIds?: string[];
   parentId?: string;
-  // TODO(nodes-core): remove offset pagination upon project cleanup
-  pagination?: CursorPaginationParams | OffsetPaginationParams;
+  pagination?: CursorPaginationParams;
   viewType: ContentNodesViewType;
 }
 
 interface GetContentNodesForDataSourceViewResult {
   nodes: DataSourceViewContentNode[];
   total: number;
+  nextPageCursor: string | null;
 }
 
 function filterNodesByViewType(
@@ -164,35 +160,28 @@ export async function getContentNodesForDataSourceView(
         ? undefined
         : ROOT_PARENT_ID);
 
-  // TODO(nodes-core): remove offset pagination upon project cleanup
-  let nextPageCursor: string | null = pagination
-    ? isCursorPaginationParams(pagination)
-      ? pagination.cursor
-      : null
-    : null;
+  let nextPageCursor: string | null = pagination ? pagination.cursor : null;
 
   let resultNodes: CoreAPIContentNode[] = [];
-  do {
-    const coreRes = await coreAPI.searchNodes({
-      filter: {
-        data_source_views: [makeCoreDataSourceViewFilter(dataSourceView)],
-        node_ids,
-        parent_id,
-      },
-      options: { limit, cursor: nextPageCursor ?? undefined },
-    });
+  const coreRes = await coreAPI.searchNodes({
+    filter: {
+      data_source_views: [makeCoreDataSourceViewFilter(dataSourceView)],
+      node_ids,
+      parent_id,
+    },
+    options: { limit, cursor: nextPageCursor ?? undefined },
+  });
 
-    if (coreRes.isErr()) {
-      return new Err(new Error(coreRes.error.message));
-    }
+  if (coreRes.isErr()) {
+    return new Err(new Error(coreRes.error.message));
+  }
 
-    const filteredNodes = removeCatchAllFoldersIfEmpty(
-      filterNodesByViewType(coreRes.value.nodes, viewType)
-    );
+  const filteredNodes = removeCatchAllFoldersIfEmpty(
+    filterNodesByViewType(coreRes.value.nodes, viewType)
+  );
 
-    resultNodes = [...resultNodes, ...filteredNodes].slice(0, limit);
-    nextPageCursor = coreRes.value.next_page_cursor;
-  } while (nextPageCursor && resultNodes.length < limit);
+  resultNodes = [...resultNodes, ...filteredNodes].slice(0, limit);
+  nextPageCursor = coreRes.value.next_page_cursor;
 
   return new Ok({
     nodes: resultNodes.map((node) =>

front/lib/api/pagination.ts

@@ -84,41 +84,41 @@ export function getPaginationParams(
   return new Ok(queryValidation.right);
 }
 
-// Offset pagination.
+// Cursor pagination.
 
-const OffsetPaginationParamsCodec = t.type({
+const CursorPaginationParamsCodec = t.type({
   limit: LimitCodec,
-  offset: t.number,
+  cursor: t.string,
 });
 
-export interface OffsetPaginationParams {
+export interface CursorPaginationParams {
   limit: number;
-  offset: number;
+  cursor: string | null;
 }
 
-export function getOffsetPaginationParams(
+export function getCursorPaginationParams(
   req: NextApiRequest
-): Result<OffsetPaginationParams | undefined, InvalidPaginationParamsError> {
-  const { limit, offset } = req.query;
+): Result<CursorPaginationParams | undefined, InvalidPaginationParamsError> {
+  const { limit, cursor } = req.query;
   if (!req.query.limit) {
     return new Ok(undefined);
   }
 
-  if (typeof limit !== "string" || (offset && typeof offset !== "string")) {
+  if (typeof limit !== "string" || (cursor && typeof cursor !== "string")) {
     return new Err(
       new InvalidPaginationParamsError(
         "Invalid pagination parameters",
-        "limit and offset must be strings"
+        "limit and cursor must be strings"
       )
     );
   }
 
   const rawParams = {
-    limit: parseInt(limit),
-    offset: offset ? parseInt(offset) : 0,
+    limit: parseInt(limit, 10),
+    cursor,
   };
 
-  const queryValidation = OffsetPaginationParamsCodec.decode(rawParams);
+  const queryValidation = CursorPaginationParamsCodec.decode(rawParams);
 
   // Validate and decode the raw parameters.
   if (isLeft(queryValidation)) {
@@ -134,14 +134,3 @@ export function getOffsetPaginationParams(
 
   return new Ok(queryValidation.right);
 }
-
-export interface CursorPaginationParams {
-  limit: number;
-  cursor: string;
-}
-
-export function isCursorPaginationParams(
-  pagination: CursorPaginationParams | OffsetPaginationParams
-): pagination is CursorPaginationParams {
-  return "cursor" in pagination;
-}

front/lib/swr/data_source_views.ts

@@ -3,12 +3,11 @@ import type {
   DataSourceViewType,
   LightWorkspaceType,
 } from "@dust-tt/types";
-import type { PaginationState } from "@tanstack/react-table";
 import { useMemo } from "react";
 import type { Fetcher, KeyedMutator, SWRConfiguration } from "swr";
 
+import type { CursorPaginationParams } from "@app/lib/api/pagination";
 import {
-  appendPaginationParams,
   fetcher,
   fetcherMultiple,
   fetcherWithBody,
@@ -113,7 +112,7 @@ export function useDataSourceViewContentNodes({
   dataSourceView?: DataSourceViewType;
   internalIds?: string[];
   parentId?: string;
-  pagination?: PaginationState;
+  pagination?: CursorPaginationParams;
   viewType?: ContentNodesViewType;
   disabled?: boolean;
   swrOptions?: SWRConfiguration;
@@ -125,9 +124,15 @@ export function useDataSourceViewContentNodes({
   mutateRegardlessOfQueryParams: KeyedMutator<GetDataSourceViewContentNodes>;
   nodes: GetDataSourceViewContentNodes["nodes"];
   totalNodesCount: number;
+  nextPageCursor: string | null;
 } {
   const params = new URLSearchParams();
-  appendPaginationParams(params, pagination);
+  if (pagination?.cursor) {
+    params.append("cursor", pagination.cursor);
+  }
+  if (pagination?.limit) {
+    params.append("limit", pagination.limit.toString());
+  }
 
   const url = dataSourceView
     ? `/api/w/${owner.sId}/spaces/${dataSourceView.spaceId}/data_source_views/${dataSourceView.sId}/content-nodes?${params}`
@@ -169,6 +174,7 @@ export function useDataSourceViewContentNodes({
     mutateRegardlessOfQueryParams,
     nodes: useMemo(() => (data ? data.nodes : []), [data]),
     totalNodesCount: data ? data.total : 0,
+    nextPageCursor: data?.nextPageCursor || null,
   };
 }
 

front/pages/api/poke/workspaces/[wId]/spaces/[spaceId]/data_source_views/[dsvId]/content-nodes.ts

@@ -10,7 +10,7 @@ import type { NextApiRequest, NextApiResponse } from "next";
 
 import { withSessionAuthentication } from "@app/lib/api/auth_wrappers";
 import { getContentNodesForDataSourceView } from "@app/lib/api/data_source_view";
-import { getOffsetPaginationParams } from "@app/lib/api/pagination";
+import { getCursorPaginationParams } from "@app/lib/api/pagination";
 import { Authenticator } from "@app/lib/auth";
 import type { SessionWithUser } from "@app/lib/iam/provider";
 import { DataSourceViewResource } from "@app/lib/resources/data_source_view_resource";
@@ -122,7 +122,7 @@ async function handler(
     });
   }
 
-  const paginationRes = getOffsetPaginationParams(req);
+  const paginationRes = getCursorPaginationParams(req);
   if (paginationRes.isErr()) {
     return apiError(req, res, {
       status_code: 400,