Remove useDataSources and associated endpoint (#9757)

front/components/spaces/EditSpaceStaticDatasourcesViews.tsx

@@ -1,13 +1,9 @@
-import { Button, PlusIcon, Popup, Tooltip } from "@dust-tt/sparkle";
+import { Button, PlusIcon, Tooltip } from "@dust-tt/sparkle";
 import type {
-  DataSourceType,
   DataSourceViewType,
-  PlanType,
   SpaceType,
   WorkspaceType,
 } from "@dust-tt/types";
-import { useRouter } from "next/router";
-import { useCallback, useState } from "react";
 
 import SpaceFolderModal from "@app/components/spaces/SpaceFolderModal";
 import SpaceWebsiteModal from "@app/components/spaces/SpaceWebsiteModal";
@@ -16,78 +12,45 @@ import { useKillSwitches } from "@app/lib/swr/kill";
 interface EditSpaceStaticDatasourcesViewsProps {
   canWriteInSpace: boolean;
   category: "folder" | "website";
-  dataSources: DataSourceType[];
   dataSourceView: DataSourceViewType | null;
   isOpen: boolean;
   onClose: () => void;
   onOpen: () => void;
   owner: WorkspaceType;
-  plan: PlanType;
   space: SpaceType;
 }
 
 export function EditSpaceStaticDatasourcesViews({
   canWriteInSpace,
   category,
-  dataSources,
   dataSourceView,
   isOpen,
   onClose,
   onOpen,
   owner,
-  plan,
   space,
 }: EditSpaceStaticDatasourcesViewsProps) {
-  const router = useRouter();
-  const [showDatasourceLimitPopup, setShowDatasourceLimitPopup] =
-    useState(false);
   const { killSwitches } = useKillSwitches();
 
   const isSavingDisabled = killSwitches?.includes("save_data_source_views");
-  const planDataSourcesLimit = plan.limits.dataSources.count;
-
-  const checkLimitsAndOpenModal = useCallback(() => {
-    if (
-      planDataSourcesLimit !== -1 &&
-      dataSources.length >= planDataSourcesLimit
-    ) {
-      setShowDatasourceLimitPopup(true);
-    } else {
-      onOpen();
-    }
-  }, [dataSources.length, planDataSourcesLimit, onOpen]);
 
   const addToSpaceButton = (
     <Button
       label={`Add ${category}`}
-      onClick={checkLimitsAndOpenModal}
+      onClick={onOpen}
       icon={PlusIcon}
       disabled={!canWriteInSpace || isSavingDisabled}
     />
   );
 
   return (
     <>
-      <Popup
-        show={showDatasourceLimitPopup}
-        chipLabel={`${plan.name} plan`}
-        description={`You have reached the limit of data sources (${plan.limits.dataSources.count} data sources). Upgrade your plan for unlimited datasources.`}
-        buttonLabel="Check Dust plans"
-        buttonClick={() => {
-          void router.push(`/w/${owner.sId}/subscription`);
-        }}
-        onClose={() => {
-          setShowDatasourceLimitPopup(false);
-        }}
-        className="absolute bottom-8 right-0"
-      />
       {category === "folder" ? (
         <SpaceFolderModal
           isOpen={isOpen}
           onClose={onClose}
           owner={owner}
           space={space}
-          dataSources={dataSources}
           dataSourceViewId={dataSourceView ? dataSourceView.sId : null}
         />
       ) : category === "website" ? (

front/components/spaces/FoldersHeaderMenu.tsx

@@ -21,7 +21,6 @@ import { useState } from "react";
 
 import type { ContentActionsRef } from "@app/components/spaces/ContentActions";
 import SpaceFolderModal from "@app/components/spaces/SpaceFolderModal";
-import { useDataSources } from "@app/lib/swr/data_sources";
 
 interface FoldersHeaderMenuProps {
   canWriteInSpace: boolean;
@@ -151,8 +150,6 @@ const EditFolderButton = ({
   owner,
   space,
 }: EditFolderButtonProps) => {
-  const { dataSources } = useDataSources(owner);
-
   const [showEditFolderModal, setShowEditFolderModal] = useState(false);
 
   return (
@@ -164,7 +161,6 @@ const EditFolderButton = ({
         }}
         owner={owner}
         space={space}
-        dataSources={dataSources}
         dataSourceViewId={folder.sId}
       />
       <Button

front/components/spaces/SpaceFolderModal.tsx

@@ -6,11 +6,7 @@ import {
   Spinner,
   TextArea,
 } from "@dust-tt/sparkle";
-import type {
-  DataSourceType,
-  LightWorkspaceType,
-  SpaceType,
-} from "@dust-tt/types";
+import type { LightWorkspaceType, SpaceType } from "@dust-tt/types";
 import { isDataSourceNameValid } from "@dust-tt/types";
 import { useRouter } from "next/router";
 import { useEffect, useState } from "react";
@@ -24,7 +20,6 @@ import {
 } from "@app/lib/swr/spaces";
 
 interface SpaceFolderModalProps {
-  dataSources: DataSourceType[];
   dataSourceViewId: string | null;
   isOpen: boolean;
   onClose: () => void;
@@ -33,7 +28,6 @@ interface SpaceFolderModalProps {
 }
 
 export default function SpaceFolderModal({
-  dataSources,
   dataSourceViewId,
   isOpen,
   onClose,
@@ -90,11 +84,6 @@ export default function SpaceFolderModal({
       nameError = "Name is required.";
     } else if (isDataSourceNameValid(name).isErr()) {
       nameError = "Name is invalid, must be multiple characters with no space.";
-    } else if (
-      (!dataSourceView || dataSourceView.dataSource.name !== name) &&
-      dataSources.find((ds) => ds.name === name)
-    ) {
-      nameError = "A data source with this name already exists.";
     }
 
     if (nameError) {

front/components/spaces/SpaceResourcesList.tsx

@@ -48,7 +48,6 @@ import { UsedByButton } from "@app/components/spaces/UsedByButton";
 import { getConnectorProviderLogoWithFallback } from "@app/lib/connector_providers";
 import { getDataSourceNameFromView, isManaged } from "@app/lib/data_sources";
 import { useAgentConfigurationSIdLookup } from "@app/lib/swr/assistants";
-import { useDataSources } from "@app/lib/swr/data_sources";
 import {
   useDeleteFolderOrWebsite,
   useSpaceDataSourceViewsWithDetails,
@@ -278,7 +277,6 @@ export const SpaceResourcesList = ({
   const [sorting, setSorting] = React.useState<SortingState>([
     { id: "name", desc: false },
   ]);
-  const { dataSources, isDataSourcesLoading } = useDataSources(owner);
   const router = useRouter();
 
   const searchBarRef = useRef<HTMLInputElement>(null);
@@ -395,11 +393,7 @@ export const SpaceResourcesList = ({
     }
   }, [selectedDataSourceView, doDelete, router, owner.sId, space.sId]);
 
-  if (
-    isDataSourcesLoading ||
-    isSpaceDataSourceViewsLoading ||
-    isNewConnectorLoading
-  ) {
+  if (isSpaceDataSourceViewsLoading || isNewConnectorLoading) {
     return (
       <div className="mt-8 flex justify-center">
         <Spinner size="lg" />
@@ -505,9 +499,7 @@ export const SpaceResourcesList = ({
               owner={owner}
               space={space}
               canWriteInSpace={canWriteInSpace}
-              dataSources={dataSources}
               dataSourceView={selectedDataSourceView}
-              plan={plan}
               category={category}
               onClose={() => {
                 setShowFolderOrWebsiteModal(false);

front/components/spaces/SpaceWebsiteModal.tsx

@@ -45,7 +45,6 @@ import { useCallback, useEffect, useState } from "react";
 
 import { DeleteStaticDataSourceDialog } from "@app/components/data_source/DeleteStaticDataSourceDialog";
 import { useDataSourceViewConnectorConfiguration } from "@app/lib/swr/data_source_views";
-import { useDataSources } from "@app/lib/swr/data_sources";
 import { useSpaceDataSourceViews } from "@app/lib/swr/spaces";
 import { urlToDataSourceName } from "@app/lib/webcrawler";
 import type { PostDataSourceWithProviderRequestBodySchema } from "@app/pages/api/w/[wId]/spaces/[spaceId]/data_sources";
@@ -89,8 +88,6 @@ export default function SpaceWebsiteModal({
     null
   );
 
-  const { dataSources, mutateDataSources } = useDataSources(owner);
-
   const [maxPages, setMaxPages] = useState<number | null>(
     WEBCRAWLER_DEFAULT_CONFIGURATION.maxPageToCrawl
   );
@@ -204,14 +201,8 @@ export default function SpaceWebsiteModal({
 
     // Validate Name (if it's not edition)
     if (!webCrawlerConfiguration) {
-      const nameExists = dataSources.some(
-        (d) =>
-          d.name === dataSourceName && d.sId !== dataSourceView?.dataSource.sId
-      );
       const dataSourceNameRes = isDataSourceNameValid(dataSourceName);
-      if (nameExists) {
-        nameError = "A Website with the same name already exists";
-      } else if (!dataSourceName.length) {
+      if (!dataSourceName.length) {
         nameError = "Please provide a name.";
       } else if (dataSourceNameRes.isErr()) {
         nameError = dataSourceNameRes.error;
@@ -221,13 +212,7 @@ export default function SpaceWebsiteModal({
     setDataSourceUrlError(urlError);
     setDataSourceNameError(nameError);
     return !urlError && !nameError;
-  }, [
-    dataSourceUrl,
-    dataSources,
-    dataSourceName,
-    dataSourceView?.dataSource.sId,
-    webCrawlerConfiguration,
-  ]);
+  }, [dataSourceUrl, dataSourceName, webCrawlerConfiguration]);
 
   useEffect(() => {
     if (isSubmitted) {
@@ -331,7 +316,6 @@ export default function SpaceWebsiteModal({
         description: `The website has been successfully ${action}.`,
       });
       void mutateSpaceDataSourceViews();
-      void mutateDataSources();
       setIsSaving(false);
     } else {
       const err: { error: APIError } = await res.json();

front/lib/swr/data_sources.ts

@@ -3,29 +3,8 @@ import { useMemo } from "react";
 import type { Fetcher } from "swr";
 
 import { fetcher, useSWRWithDefaults } from "@app/lib/swr/swr";
-import type { GetDataSourcesResponseBody } from "@app/pages/api/w/[wId]/data_sources";
 import type { GetDataSourceUsageResponseBody } from "@app/pages/api/w/[wId]/data_sources/[dsId]/usage";
 
-export function useDataSources(
-  owner: LightWorkspaceType,
-  options = { disabled: false }
-) {
-  const { disabled } = options;
-  const dataSourcesFetcher: Fetcher<GetDataSourcesResponseBody> = fetcher;
-  const { data, error, mutate } = useSWRWithDefaults(
-    `/api/w/${owner.sId}/data_sources`,
-    dataSourcesFetcher,
-    { disabled }
-  );
-
-  return {
-    dataSources: useMemo(() => (data ? data.dataSources : []), [data]),
-    isDataSourcesLoading: disabled ? false : !error && !data,
-    isDataSourcesError: disabled ? false : error,
-    mutateDataSources: mutate,
-  };
-}
-
 export function useDataSourceUsage({
   owner,
   dataSource,