@@ -525,300 +462,3 @@ const DocumentUploadOrEditModal = ({
);
};
-
-interface Table {
- name: string;
- description: string;
- file: File | null;
-}
-
-const TableUploadOrEditModal = ({
- initialId,
- dataSourceView,
- isOpen,
- onClose,
- owner,
-}: DocumentOrTableUploadOrEditModalProps) => {
- const sendNotification = useSendNotification();
- const fileInputRef = useRef
(null);
-
- const [tableState, setTableState] = useState({
- name: "",
- description: "",
- file: null,
- });
- const [editionStatus, setEditionStatus] = useState({
- name: false,
- description: false,
- });
- const [isUpserting, setIsUpserting] = useState(false);
- const [isBigFile, setIsBigFile] = useState(false);
- const [isValidTable, setIsValidTable] = useState(false);
- const [useAppForHeaderDetection, setUseAppForHeaderDetection] =
- useState(false);
-
- const { table, isTableError, isTableLoading } = useTable({
- owner: owner,
- dataSourceView: dataSourceView,
- tableId: initialId ?? null,
- });
-
- const { featureFlags } = useFeatureFlags({ workspaceId: owner.sId });
-
- useEffect(() => {
- if (!initialId) {
- setTableState({
- name: "",
- description: "",
- file: null,
- });
- } else if (table) {
- setTableState((prev) => ({
- ...prev,
- name: table.name,
- description: table.description,
- }));
- }
- }, [initialId, table]);
-
- useEffect(() => {
- const isNameValid = !!tableState.name && isSlugified(tableState.name);
- const isContentValid = !!tableState.description;
- setIsValidTable(isNameValid && isContentValid && !!tableState.file);
- }, [tableState]);
-
- const handleTableUpload = async (table: Table) => {
- setIsUpserting(true);
- try {
- const fileContent = table.file
- ? await handleFileUploadToText(table.file)
- : null;
- if (fileContent && fileContent.isErr()) {
- return new Err(fileContent.error);
- }
-
- const csvContent = fileContent?.value
- ? await parseAndStringifyCsv(fileContent.value.content)
- : null;
-
- if (csvContent && csvContent.length > MAX_FILE_LENGTH) {
- throw new Error("File too large");
- }
-
- const base = `/api/w/${owner.sId}/spaces/${dataSourceView.spaceId}/data_sources/${dataSourceView.dataSource.sId}/tables`;
- const endpoint = initialId ? `${base}/${initialId}` : base;
-
- const body = JSON.stringify({
- name: table.name,
- description: table.description,
- csv: csvContent,
- tableId: initialId,
- timestamp: null,
- tags: [],
- parents: [],
- truncate: true,
- async: false,
- useAppForHeaderDetection,
- });
-
- const res = await fetch(endpoint, {
- method: initialId ? "PATCH" : "POST",
- headers: { "Content-Type": "application/json" },
- body,
- });
-
- if (!res.ok) {
- throw new Error("Failed to upsert table");
- }
-
- sendNotification({
- type: "success",
- title: `Table successfully ${initialId ? "updated" : "added"}`,
- description: `Table ${table.name} was successfully ${initialId ? "updated" : "added"}.`,
- });
- } catch (error) {
- sendNotification({
- type: "error",
- title: "Error upserting table",
- description: `An error occurred: ${error instanceof Error ? error.message : String(error)}.`,
- });
- } finally {
- setIsUpserting(false);
- }
- };
-
- const handleUpload = async () => {
- try {
- await handleTableUpload(tableState);
- onClose(true);
- } catch (error) {
- console.error(error);
- }
- };
-
- const handleFileChange = async (e: React.ChangeEvent) => {
- const selectedFile = e.target.files?.[0];
- if (!selectedFile) {
- return;
- }
-
- setIsUpserting(true);
- try {
- if (selectedFile.size > MAX_FILE_SIZES.plainText) {
- sendNotification({
- type: "error",
- title: "File too large",
- description: `Please upload a file smaller than ${maxFileSizeToHumanReadable(MAX_FILE_SIZES.plainText)}.`,
- });
- setIsUpserting(false);
- return;
- }
-
- setTableState((prev) => ({ ...prev, file: selectedFile }));
- setIsBigFile(selectedFile.size > BIG_FILE_SIZE);
- } catch (error) {
- sendNotification({
- type: "error",
- title: "Error uploading file",
- description: error instanceof Error ? error.message : String(error),
- });
- } finally {
- setIsUpserting(false);
- }
- };
-
- return (
- {
- onClose(false);
- }}
- hasChanged={!isTableError && !isTableLoading && isValidTable}
- variant="side-md"
- title={`${initialId ? "Edit" : "Add"} table`}
- onSave={handleUpload}
- isSaving={isUpserting}
- >
- {isTableLoading ? (
-
-
-
- ) : (
-
- {isTableError ? (
- Content cannot be loaded.
- ) : (
-
-
-
-
{
- setEditionStatus((prev) => ({ ...prev, name: true }));
- setTableState((prev) => ({
- ...prev,
- name: e.target.value,
- }));
- }}
- message={
- editionStatus.name &&
- (!tableState.name || !isSlugified(tableState.name))
- ? "Invalid name: Must be alphanumeric, max 32 characters and no space."
- : null
- }
- messageStatus="error"
- />
-
-
-
-
-
-
fileInputRef.current?.click(),
- }}
- />
-
- {isBigFile && (
-
-
-
- Warning: Large file (5MB+)
-
-
- This file is large and may take a while to upload.
-
-
- )}
-
- {featureFlags.includes("use_app_for_header_detection") && (
-
-
- setUseAppForHeaderDetection(!useAppForHeaderDetection),
- }}
- />
-
- )}
-
- )}
-
- )}
-
- );
-};
diff --git a/front/components/data_source/MultipleDocumentsUpload.tsx b/front/components/data_source/MultipleDocumentsUpload.tsx
index b732e36ecb59..8582f0f35d4c 100644
--- a/front/components/data_source/MultipleDocumentsUpload.tsx
+++ b/front/components/data_source/MultipleDocumentsUpload.tsx
@@ -3,16 +3,19 @@ import type {
DataSourceViewType,
LightWorkspaceType,
PlanType,
- PostDataSourceWithNameDocumentRequestBody,
} from "@dust-tt/types";
-import { Err, Ok } from "@dust-tt/types";
+import { Err, supportedPlainTextExtensions } from "@dust-tt/types";
import type { ChangeEvent } from "react";
import { useCallback, useEffect, useRef, useState } from "react";
import { DocumentLimitPopup } from "@app/components/data_source/DocumentLimitPopup";
-import { handleFileUploadToText } from "@app/lib/client/handle_file_upload";
-
-const UPLOAD_ACCEPT = [".txt", ".pdf", ".md", ".csv"];
+import type {
+ FileBlob,
+ FileBlobWithFileId,
+} from "@app/hooks/useFileUploaderService";
+import { useFileUploaderService } from "@app/hooks/useFileUploaderService";
+import { useCreateDataSourceViewDocument } from "@app/lib/swr/data_source_view_documents";
+import { getFileProcessedUrl } from "@app/lib/swr/file";
type MultipleDocumentsUploadProps = {
dataSourceView: DataSourceViewType;
@@ -31,15 +34,11 @@ export const MultipleDocumentsUpload = ({
totalNodesCount,
plan,
}: MultipleDocumentsUploadProps) => {
+ const sendNotification = useSendNotification();
const fileInputRef = useRef(null);
const [isLimitPopupOpen, setIsLimitPopupOpen] = useState(false);
const [wasOpened, setWasOpened] = useState(isOpen);
- const [isBulkFilesUploading, setIsBulkFilesUploading] = useState(null);
-
const close = useCallback(
(save: boolean) => {
// Clear the values of the file input
@@ -51,117 +50,123 @@ export const MultipleDocumentsUpload = ({
[onClose]
);
- const sendNotification = useSendNotification();
-
- const handleUpsert = useCallback(
- async (text: string, documentId: string) => {
- const body: PostDataSourceWithNameDocumentRequestBody = {
- name: documentId,
- timestamp: null,
- parents: null,
- section: {
- prefix: null,
- content: text,
- sections: [],
- },
- text: null,
- source_url: undefined,
- tags: [],
- light_document_output: true,
- upsert_context: null,
- async: false,
- };
-
- try {
- const res = await fetch(
- `/api/w/${owner.sId}/spaces/${dataSourceView.spaceId}/data_sources/${
- dataSourceView.dataSource.sId
- }/documents`,
- {
- method: "POST",
- headers: {
- "Content-Type": "application/json",
- },
- body: JSON.stringify(body),
- }
- );
-
- if (!res.ok) {
- let errMsg = "";
- try {
- const data = await res.json();
- errMsg = data.error.message;
- } catch (e) {
- errMsg = "An error occurred while uploading your document.";
- }
- return new Err(errMsg);
- }
- } catch (e) {
- return new Err("An error occurred while uploading your document.");
+ const getFileProcessedContent = useCallback(
+ async (fileId: string) => {
+ const url = getFileProcessedUrl(owner, fileId);
+ const res = await fetch(url);
+ if (!res.ok) {
+ return new Err(`Error reading the file content: ${res.status}`);
}
-
- return new Ok(null);
+ const content = await res.text();
+ if (content === null || content === "") {
+ return new Err("Empty file content");
+ }
+ return content;
},
- [dataSourceView.dataSource.sId, dataSourceView.spaceId, owner.sId]
+ [owner]
);
+ // Used for creating files, with text extraction post-processing
+ const fileUploaderService = useFileUploaderService({
+ owner,
+ useCase: "folder_document",
+ });
+
+ // Mutation for creating documents, throw error on partial failure
+ const doCreate = useCreateDataSourceViewDocument(owner, dataSourceView);
+
+ const [isBulkFilesUploading, setIsBulkFilesUploading] = useState(null);
+
const handleFileChange = useCallback(
async (
e: ChangeEvent & { target: { files: File[] } }
) => {
- if (e.target.files && e.target.files.length > 0) {
- if (
- plan.limits.dataSources.documents.count != -1 &&
- e.target.files.length + totalNodesCount >
- plan.limits.dataSources.documents.count
- ) {
- setIsLimitPopupOpen(true);
- return;
- }
- const files = e.target.files;
- let i = 0;
- for (const file of files) {
- setIsBulkFilesUploading({
- total: files.length,
- completed: i++,
- });
- try {
- const uploadRes = await handleFileUploadToText(file);
- if (uploadRes.isErr()) {
- sendNotification({
- type: "error",
- title: `Error uploading document ${file.name}`,
- description: uploadRes.error.message,
- });
- } else {
- const upsertRes = await handleUpsert(
- uploadRes.value.content,
- file.name
- );
- if (upsertRes.isErr()) {
- sendNotification({
- type: "error",
- title: `Error uploading document ${file.name}`,
- description: upsertRes.error,
- });
- }
- }
- } catch (e) {
- sendNotification({
- type: "error",
- title: "Error uploading document",
- description: `An error occurred while uploading your documents.`,
- });
- }
- }
+ // Empty file input
+ if (!e.target.files || e.target.files.length === 0) {
+ close(false);
+ return;
+ }
+
+ // Open plan popup if limit is reached
+ if (
+ plan.limits.dataSources.documents.count != -1 &&
+ e.target.files.length + totalNodesCount >
+ plan.limits.dataSources.documents.count
+ ) {
+ setIsLimitPopupOpen(true);
+ return;
+ }
+
+ setIsBulkFilesUploading({
+ total: e.target.files.length,
+ completed: 0,
+ });
+
+ // upload Files and get FileBlobs (only keep successfull uploads)
+ // Each individual error triggers a notification
+ const fileBlobs = (await fileUploaderService.handleFileChange(e))?.filter(
+ (fileBlob: FileBlob): fileBlob is FileBlobWithFileId =>
+ !!fileBlob.fileId
+ );
+ if (!fileBlobs || fileBlobs.length === 0) {
setIsBulkFilesUploading(null);
- close(true);
- } else {
close(false);
+ fileUploaderService.resetUpload();
+ return;
+ }
+
+ // upsert the file as Data Source Documents
+ // Done 1 by 1 for simplicity
+ let i = 0;
+ for (const blob of fileBlobs) {
+ setIsBulkFilesUploading({
+ total: fileBlobs.length,
+ completed: i++,
+ });
+
+ // get processed text
+ const content = await getFileProcessedContent(blob.fileId);
+ if (content instanceof Err) {
+ sendNotification({
+ type: "error",
+ title: `Error processing document ${blob.filename}`,
+ description: content.error,
+ });
+ continue;
+ }
+
+ // Create the document
+ const body = {
+ name: blob.filename,
+ timestamp: null,
+ parents: [blob.filename],
+ section: {
+ prefix: null,
+ content: content,
+ sections: [],
+ },
+ text: null,
+ source_url: undefined,
+ tags: [],
+ light_document_output: true,
+ upsert_context: null,
+ async: false,
+ };
+ await doCreate(body);
}
+
+ // Reset the upload state
+ setIsBulkFilesUploading(null);
+ fileUploaderService.resetUpload();
+ close(true);
},
[
- handleUpsert,
+ doCreate,
+ fileUploaderService,
+ getFileProcessedContent,
close,
plan.limits.dataSources.documents.count,
sendNotification,
@@ -173,6 +178,7 @@ export const MultipleDocumentsUpload = ({
close(false);
}, [close]);
+ // Effect: add event listener to file input
useEffect(() => {
const ref = fileInputRef.current;
ref?.addEventListener("cancel", handleFileInputBlur);
@@ -181,6 +187,7 @@ export const MultipleDocumentsUpload = ({
};
}, [handleFileInputBlur]);
+ // Effect: open file input when the dialog is opened
useEffect(() => {
if (isOpen && !wasOpened) {
const ref = fileInputRef.current;
@@ -220,7 +227,7 @@ export const MultipleDocumentsUpload = ({
void;
+ owner: WorkspaceType;
+ plan: PlanType;
+ totalNodesCount: number;
+ initialId?: string;
+}
+const MAX_NAME_CHARS = 32;
+
+export const TableUploadOrEditModal = ({
+ initialId,
+ dataSourceView,
+ isOpen,
+ onClose,
+ owner,
+}: TableUploadOrEditModalProps) => {
+ const sendNotification = useSendNotification();
+ const fileInputRef = useRef(null);
+
+ const [tableState, setTableState] = useState({
+ name: "",
+ description: "",
+ file: null,
+ content: null,
+ });
+ const [editionStatus, setEditionStatus] = useState({
+ name: false,
+ description: false,
+ });
+ const [isUpserting, setIsUpserting] = useState(false);
+ const [isBigFile, setIsBigFile] = useState(false);
+ const [isValidTable, setIsValidTable] = useState(false);
+ const [useAppForHeaderDetection, setUseAppForHeaderDetection] =
+ useState(false);
+ const { table, isTableError, isTableLoading } = useDataSourceViewTable({
+ owner: owner,
+ dataSourceView: dataSourceView,
+ tableId: initialId ?? null,
+ disabled: !initialId,
+ });
+
+ // Get the processed file content from the file API
+ const fileUploaderService = useFileUploaderService({
+ owner,
+ useCase: "folder_table",
+ });
+ const [fileId, setFileId] = useState(null);
+ const { isContentLoading } = useFileProcessedContent(owner, fileId ?? null, {
+ disabled: !fileId,
+ onSuccess: async (response) => {
+ const content = await response.text();
+ setTableState((prev) => ({
+ ...prev,
+ content: content ?? null,
+ }));
+ },
+ onError: (error) => {
+ fileUploaderService.resetUpload();
+ sendNotification({
+ type: "error",
+ title: "Error fetching content",
+ description: error instanceof Error ? error.message : String(error),
+ });
+ },
+ shouldRetryOnError: false,
+ });
+ const { featureFlags } = useFeatureFlags({ workspaceId: owner.sId });
+
+ // Mutations for upserting the table
+ const doUpdate = useUpdateDataSourceViewTable(
+ owner,
+ dataSourceView,
+ initialId ?? ""
+ );
+ const doCreate = useCreateDataSourceViewTable(owner, dataSourceView);
+
+ const handleTableUpload = useCallback(
+ async (table: Table) => {
+ setIsUpserting(true);
+
+ if (!table.content) {
+ sendNotification({
+ type: "error",
+ title: "No content",
+ description: "No content to upload.",
+ });
+ setIsUpserting(false);
+ return;
+ }
+
+ try {
+ const body = {
+ name: table.name,
+ description: table.description,
+ csv: table.content,
+ tableId: initialId,
+ timestamp: null,
+ tags: [],
+ parents: [],
+ truncate: true,
+ async: false,
+ useAppForHeaderDetection,
+ };
+ let upsertRes = null;
+ if (initialId) {
+ upsertRes = await doUpdate(body);
+ } else {
+ upsertRes = await doCreate(body);
+ }
+
+ // Upsert successful, close and reset the modal
+ if (upsertRes) {
+ onClose(true);
+ setTableState({
+ name: "",
+ description: "",
+ file: null,
+ content: null,
+ });
+ setEditionStatus({
+ description: false,
+ name: false,
+ });
+ }
+
+ // No matter the result, reset the file uploader
+ setFileId(null);
+ fileUploaderService.resetUpload();
+ } catch (error) {
+ console.error(error);
+ } finally {
+ setIsUpserting(false);
+ }
+ },
+ [
+ initialId,
+ onClose,
+ sendNotification,
+ doCreate,
+ doUpdate,
+ fileUploaderService,
+ useAppForHeaderDetection,
+ ]
+ );
+
+ const handleUpload = async () => {
+ try {
+ await handleTableUpload(tableState);
+ onClose(true);
+ } catch (error) {
+ console.error(error);
+ }
+ };
+
+ const handleFileChange = async (e: React.ChangeEvent) => {
+ // Enforce single file upload
+ const files = e.target.files;
+ if (files && files.length > 1) {
+ sendNotification({
+ type: "error",
+ title: "Multiple files",
+ description: "Please upload only one file at a time.",
+ });
+ return;
+ }
+
+ try {
+ // Create a file -> Allows to get processed text content via the file API.
+ const selectedFile = files?.[0];
+ if (!selectedFile) {
+ return;
+ }
+ const fileBlobs = await fileUploaderService.handleFilesUpload([
+ selectedFile,
+ ]);
+ if (!fileBlobs || fileBlobs.length == 0 || !fileBlobs[0].fileId) {
+ fileUploaderService.resetUpload();
+ return new Err(
+ new Error(
+ "Error uploading file. Please try again or contact support."
+ )
+ );
+ }
+
+ // triggers content extraction -> tableState.content update
+ setFileId(fileBlobs[0].fileId);
+ setTableState((prev) => ({
+ ...prev,
+ file: selectedFile,
+ name:
+ prev.name.length > 0 ? prev.name : stripTableName(selectedFile.name),
+ }));
+ setIsBigFile(selectedFile.size > BIG_FILE_SIZE);
+ } catch (error) {
+ sendNotification({
+ type: "error",
+ title: "Error uploading file",
+ description: error instanceof Error ? error.message : String(error),
+ });
+ } finally {
+ e.target.value = "";
+ fileUploaderService.resetUpload();
+ }
+ };
+
+ // Effect: Validate the table state when inputs change
+ useEffect(() => {
+ const isNameValid = !!tableState.name && isSlugified(tableState.name);
+ const isContentValid = !!tableState.description;
+ setIsValidTable(isNameValid && isContentValid && !!tableState.content);
+ }, [tableState]);
+
+ // Effect: Set the table state when the table is loaded
+ useEffect(() => {
+ if (!initialId) {
+ setTableState({
+ name: "",
+ description: "",
+ file: null,
+ content: null,
+ });
+ } else if (table) {
+ setTableState((prev) => ({
+ ...prev,
+ name: table.name,
+ description: table.description,
+ }));
+ }
+ }, [initialId, table]);
+
+ return (
+ {
+ onClose(false);
+ }}
+ hasChanged={!isTableError && !isTableLoading && isValidTable}
+ variant="side-md"
+ title={`${initialId ? "Edit" : "Add"} table`}
+ onSave={handleUpload}
+ isSaving={isUpserting}
+ >
+ {isTableLoading ? (
+
+
+
+ ) : (
+
+ {isTableError ? (
+ Content cannot be loaded.
+ ) : (
+
+
+
+
{
+ setEditionStatus((prev) => ({ ...prev, name: true }));
+ setTableState((prev) => ({
+ ...prev,
+ name: e.target.value,
+ }));
+ }}
+ message={
+ editionStatus.name &&
+ (!tableState.name || !isSlugified(tableState.name))
+ ? "Invalid name: Must be alphanumeric, max 32 characters and no space."
+ : null
+ }
+ messageStatus="error"
+ />
+
+
+
+
+
+
fileInputRef.current?.click(),
+ }}
+ />
+
+ {isBigFile && (
+
+
+
+ Warning: Large file (5MB+)
+
+
+ This file is large and may take a while to upload.
+
+
+ )}
+
+ {featureFlags.includes("use_app_for_header_detection") && (
+
+
+ setUseAppForHeaderDetection(!useAppForHeaderDetection),
+ }}
+ />
+
+ )}
+
+ )}
+
+ )}
+
+ );
+};
+
+function stripTableName(name: string) {
+ return name
+ .replace(/\.(csv|tsv)$/, "")
+ .replace(/[^a-z0-9]/gi, "_")
+ .toLowerCase()
+ .slice(0, 32);
+}
diff --git a/front/components/spaces/ContentActions.tsx b/front/components/spaces/ContentActions.tsx
index 40f454d16f2d..a680d54afc2f 100644
--- a/front/components/spaces/ContentActions.tsx
+++ b/front/components/spaces/ContentActions.tsx
@@ -24,8 +24,9 @@ import React, {
} from "react";
import { DocumentOrTableDeleteDialog } from "@app/components/data_source/DocumentOrTableDeleteDialog";
-import { DocumentOrTableUploadOrEditModal } from "@app/components/data_source/DocumentOrTableUploadOrEditModal";
+import { DocumentUploadOrEditModal } from "@app/components/data_source/DocumentUploadOrEditModal";
import { MultipleDocumentsUpload } from "@app/components/data_source/MultipleDocumentsUpload";
+import { TableUploadOrEditModal } from "@app/components/data_source/TableUploadOrEditModal";
import DataSourceViewDocumentModal from "@app/components/DataSourceViewDocumentModal";
import { AddToSpaceDialog } from "@app/components/spaces/AddToSpaceDialog";
import {
@@ -34,7 +35,6 @@ import {
isManaged,
isWebsite,
} from "@app/lib/data_sources";
-
export type UploadOrEditContentActionKey =
| "DocumentUploadOrEdit"
| "TableUploadOrEdit";
@@ -118,26 +118,30 @@ export const ContentActions = React.forwardRef<
[currentAction, onSave]
);
+ const contentNode = isUploadOrEditAction(currentAction.action)
+ ? currentAction.contentNode
+ : undefined;
+
+ // This is a union of the props for the two modals
+ // Makes sense because both expect the same schema
+ const modalProps = {
+ contentNode,
+ dataSourceView,
+ isOpen: isUploadOrEditAction(currentAction.action),
+ onClose,
+ owner,
+ plan,
+ totalNodesCount,
+ initialId: contentNode?.internalId,
+ };
+
return (
<>
-
+ {currentAction.action === "TableUploadOrEdit" ? (
+
+ ) : (
+
+ )}
f.id === file.name)) {
sendNotification({
type: "error",
- title: "File already exists.",
+ title: `Failed to upload file ${file.name}`,
description: `File "${file.name}" is already uploaded.`,
});
@@ -261,8 +261,8 @@ export function useFileUploaderService({
erroredBlobs.push(result.error);
sendNotification({
type: "error",
- title: "Failed to upload file.",
- description: result.error.message,
+ title: `Failed to upload file`,
+ description: `error uploading ${result.error.file.name} ${result.error.message ? ": " + result.error.message : ""}`,
});
} else {
successfulBlobs.push(result.value);
@@ -315,7 +315,6 @@ export function useFileUploaderService({
setFileBlobs([]);
};
- type FileBlobWithFileId = FileBlob & { fileId: string };
function fileBlobHasFileId(
fileBlob: FileBlob
): fileBlob is FileBlobWithFileId {
diff --git a/front/lib/api/files/upload.ts b/front/lib/api/files/upload.ts
index c4a31292be6c..370ec1622aaf 100644
--- a/front/lib/api/files/upload.ts
+++ b/front/lib/api/files/upload.ts
@@ -272,67 +272,78 @@ type ProcessingPerContentType = {
const processingPerContentType: ProcessingPerContentType = {
"application/msword": {
conversation: extractTextFromFileAndUpload,
- folder: extractTextFromFileAndUpload,
+ folder_document: extractTextFromFileAndUpload,
+ folder_table: notSupportedError,
avatar: notSupportedError,
tool_output: notSupportedError,
},
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": {
- folder: extractTextFromFileAndUpload,
+ folder_document: extractTextFromFileAndUpload,
+ folder_table: notSupportedError,
conversation: extractTextFromFileAndUpload,
avatar: notSupportedError,
tool_output: notSupportedError,
},
"application/pdf": {
- folder: extractTextFromFileAndUpload,
+ folder_document: extractTextFromFileAndUpload,
+ folder_table: notSupportedError,
conversation: extractTextFromFileAndUpload,
avatar: notSupportedError,
tool_output: notSupportedError,
},
"image/jpeg": {
conversation: resizeAndUploadToFileStorage,
- folder: notSupportedError,
+ folder_document: notSupportedError,
+ folder_table: notSupportedError,
avatar: uploadToPublicBucket,
tool_output: notSupportedError,
},
"image/png": {
conversation: resizeAndUploadToFileStorage,
- folder: notSupportedError,
+ folder_document: notSupportedError,
+ folder_table: notSupportedError,
avatar: uploadToPublicBucket,
tool_output: notSupportedError,
},
"text/comma-separated-values": {
conversation: extractContentAndSchemaFromCSV,
- folder: storeRawText,
+ folder_document: storeRawText,
+ folder_table: extractContentAndSchemaFromCSV,
avatar: notSupportedError,
tool_output: notSupportedError,
},
"text/csv": {
conversation: extractContentAndSchemaFromCSV,
- folder: storeRawText,
+ folder_document: storeRawText,
+ folder_table: extractContentAndSchemaFromCSV,
avatar: notSupportedError,
tool_output: notSupportedError,
},
"text/markdown": {
conversation: storeRawText,
- folder: storeRawText,
+ folder_document: storeRawText,
+ folder_table: notSupportedError,
avatar: notSupportedError,
tool_output: notSupportedError,
},
"text/plain": {
conversation: storeRawText,
- folder: storeRawText,
+ folder_document: storeRawText,
+ folder_table: notSupportedError,
avatar: notSupportedError,
tool_output: notSupportedError,
},
"text/tab-separated-values": {
conversation: storeRawText,
- folder: storeRawText,
+ folder_document: storeRawText,
+ folder_table: storeRawText,
avatar: notSupportedError,
tool_output: notSupportedError,
},
"text/tsv": {
conversation: storeRawText,
- folder: storeRawText,
+ folder_document: storeRawText,
+ folder_table: storeRawText,
avatar: notSupportedError,
tool_output: notSupportedError,
},
diff --git a/front/lib/api/files/upsert.ts b/front/lib/api/files/upsert.ts
index 4679fc21482f..4cb26842b7c1 100644
--- a/front/lib/api/files/upsert.ts
+++ b/front/lib/api/files/upsert.ts
@@ -280,55 +280,64 @@ type ProcessingPerContentType = {
const processingPerContentType: ProcessingPerContentType = {
"application/msword": {
conversation: upsertDocumentToDatasource,
- folder: notSupportedError,
+ folder_document: notSupportedError,
+ folder_table: notSupportedError,
avatar: notSupportedError,
tool_output: notSupportedError,
},
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": {
conversation: upsertDocumentToDatasource,
- folder: notSupportedError,
+ folder_document: notSupportedError,
+ folder_table: notSupportedError,
avatar: notSupportedError,
tool_output: notSupportedError,
},
"application/pdf": {
conversation: upsertDocumentToDatasource,
- folder: notSupportedError,
+ folder_document: notSupportedError,
+ folder_table: notSupportedError,
avatar: notSupportedError,
tool_output: notSupportedError,
},
"text/comma-separated-values": {
conversation: upsertTableToDatasource,
- folder: notSupportedError,
+ folder_document: notSupportedError,
+ folder_table: notSupportedError,
avatar: notSupportedError,
tool_output: upsertTableToDatasource,
},
"text/csv": {
conversation: upsertTableToDatasource,
- folder: notSupportedError,
+ folder_document: notSupportedError,
+ folder_table: notSupportedError,
avatar: notSupportedError,
tool_output: upsertTableToDatasource,
},
"text/markdown": {
conversation: upsertDocumentToDatasource,
- folder: notSupportedError,
+ folder_document: notSupportedError,
+ folder_table: notSupportedError,
avatar: notSupportedError,
tool_output: notSupportedError,
},
"text/plain": {
conversation: upsertDocumentToDatasource,
- folder: notSupportedError,
+ folder_document: notSupportedError,
+ folder_table: notSupportedError,
avatar: notSupportedError,
tool_output: notSupportedError,
},
"text/tab-separated-values": {
conversation: upsertDocumentToDatasource, // Should it be upsertTableToDatasource?
- folder: notSupportedError,
+ folder_document: notSupportedError,
+ folder_table: notSupportedError,
avatar: notSupportedError,
tool_output: notSupportedError,
},
"text/tsv": {
conversation: upsertDocumentToDatasource, // Should it be upsertTableToDatasource?
- folder: notSupportedError,
+ folder_document: notSupportedError,
+ folder_table: notSupportedError,
avatar: notSupportedError,
tool_output: notSupportedError,
},
diff --git a/front/lib/client/handle_file_upload.ts b/front/lib/client/handle_file_upload.ts
deleted file mode 100644
index 8977144bd65b..000000000000
--- a/front/lib/client/handle_file_upload.ts
+++ /dev/null
@@ -1,118 +0,0 @@
-import type { Result, SupportedContentFragmentType } from "@dust-tt/types";
-import { isSupportedTextContentFragmentType } from "@dust-tt/types";
-import { Err, Ok } from "@dust-tt/types";
-// @ts-expect-error: type package doesn't load properly because of how we are loading pdfjs
-import * as PDFJS from "pdfjs-dist/build/pdf";
-
-import { getMimeTypeFromFile } from "@app/lib/file";
-PDFJS.GlobalWorkerOptions.workerSrc = `//cdnjs.cloudflare.com/ajax/libs/pdf.js/${PDFJS.version}/pdf.worker.mjs`;
-
-interface FileBlob {
- title: string;
- content: string;
- contentType: SupportedContentFragmentType;
-}
-
-export async function extractTextFromPDF(
- file: File,
- blob: FileReader["result"]
-): Promise> {
- try {
- if (!(blob instanceof ArrayBuffer)) {
- return new Err(
- new Error("Failed extracting text from PDF. Unexpected error")
- );
- }
-
- const loadingTask = PDFJS.getDocument({ data: blob });
- const pdf = await loadingTask.promise;
-
- let text = "";
- for (let pageNum = 1; pageNum <= pdf.numPages; pageNum++) {
- const page = await pdf.getPage(pageNum);
- const content = await page.getTextContent();
- const strings = content.items.map((item: unknown) => {
- if (
- item &&
- typeof item === "object" &&
- "str" in item &&
- typeof item.str === "string"
- ) {
- return item.str;
- }
- });
- text += `Page: ${pageNum}/${pdf.numPages}\n${strings.join(" ")}\n\n`;
- }
-
- return new Ok({
- title: file.name,
- content: text,
- contentType: "application/pdf",
- });
- } catch (e) {
- console.error("Failed extracting text from PDF", e);
- const errorMessage = e instanceof Error ? e.message : "Unexpected error";
-
- return new Err(
- new Error(`Failed extracting text from PDF. ${errorMessage}`)
- );
- }
-}
-
-export async function handleFileUploadToText(
- file: File
-): Promise> {
- const contentFragmentMimeType = getMimeTypeFromFile(file);
- if (!isSupportedTextContentFragmentType(contentFragmentMimeType)) {
- return new Err(new Error("Unsupported file type."));
- }
-
- return new Promise((resolve) => {
- const handleFileLoadedText = (e: ProgressEvent) => {
- const content = e.target?.result;
- if (content && typeof content === "string") {
- return resolve(
- new Ok({
- title: file.name,
- content,
- contentType: contentFragmentMimeType,
- })
- );
- } else {
- return resolve(
- new Err(
- new Error(
- "Failed extracting text from file. Please check that your file is not empty."
- )
- )
- );
- }
- };
-
- const handleFileLoadedPDF = async (e: ProgressEvent) => {
- const { result = null } = e.target ?? {};
-
- const res = await extractTextFromPDF(file, result);
-
- return resolve(res);
- };
-
- try {
- if (contentFragmentMimeType === "application/pdf") {
- const fileReader = new FileReader();
- fileReader.onloadend = handleFileLoadedPDF;
- fileReader.readAsArrayBuffer(file);
- } else {
- const fileData = new FileReader();
- fileData.onloadend = handleFileLoadedText;
- fileData.readAsText(file);
- }
- } catch (e) {
- console.error("Error handling file", e);
- const errorMessage = e instanceof Error ? e.message : "Unexpected error";
- return resolve(
- new Err(new Error(`Error handling file. ${errorMessage}`))
- );
- }
- });
-}
diff --git a/front/lib/swr/data_source_view_documents.ts b/front/lib/swr/data_source_view_documents.ts
index 7465aef8e655..ab8f30e979be 100644
--- a/front/lib/swr/data_source_view_documents.ts
+++ b/front/lib/swr/data_source_view_documents.ts
@@ -1,54 +1,21 @@
+import { useSendNotification } from "@dust-tt/sparkle";
import type { DataSourceViewType, LightWorkspaceType } from "@dust-tt/types";
import type {
PatchDataSourceWithNameDocumentRequestBody,
PostDataSourceWithNameDocumentRequestBody,
} from "@dust-tt/types";
-import assert from "assert";
import type { Fetcher } from "swr";
-import type { SWRMutationConfiguration } from "swr/mutation";
-import useSWRMutation from "swr/mutation";
import { useDataSourceViewContentNodes } from "@app/lib/swr/data_source_views";
-import { fetcher, fetcherWithBody, useSWRWithDefaults } from "@app/lib/swr/swr";
+import {
+ fetcher,
+ getErrorFromResponse,
+ useSWRWithDefaults,
+} from "@app/lib/swr/swr";
import type { GetDataSourceViewDocumentResponseBody } from "@app/pages/api/w/[wId]/spaces/[spaceId]/data_source_views/[dsvId]/documents/[documentId]";
import type { PostDocumentResponseBody } from "@app/pages/api/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/documents";
import type { PatchDocumentResponseBody } from "@app/pages/api/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/documents/[documentId]";
-// Centralized way to get urls -> reduces key-related inconcistencies
-function getUrlHasValidParameters(
- method: "POST" | "GET" | "PATCH",
- documentId?: string
-): documentId is string {
- // Only require documentId for GET and PATCH methods
- return method === "POST" || !!documentId;
-}
-const getUrl = ({
- method,
- owner,
- dataSourceView,
- documentId,
-}: {
- method: "POST" | "GET" | "PATCH";
- owner: LightWorkspaceType;
- dataSourceView: DataSourceViewType;
- documentId?: string;
-}) => {
- assert(
- getUrlHasValidParameters(method, documentId),
- "Cannot get or patch a document without a documentId"
- );
-
- const baseUrl = `/api/w/${owner.sId}/spaces/${dataSourceView.spaceId}`;
- switch (method) {
- case "POST":
- return `${baseUrl}/data_sources/${dataSourceView.dataSource.sId}/documents`;
- case "PATCH":
- return `${baseUrl}/data_sources/${dataSourceView.dataSource.sId}/documents/${encodeURIComponent(documentId)}`;
- case "GET":
- return `${baseUrl}/data_source_views/${dataSourceView.sId}/documents/${encodeURIComponent(documentId)}`;
- }
-};
-
export function useDataSourceViewDocument({
dataSourceView,
documentId,
@@ -64,12 +31,7 @@ export function useDataSourceViewDocument({
fetcher;
const url =
dataSourceView && documentId
- ? getUrl({
- method: "GET",
- owner,
- dataSourceView,
- documentId,
- })
+ ? `/api/w/${owner.sId}/spaces/${dataSourceView.spaceId}/data_source_views/${dataSourceView.sId}/documents/${encodeURIComponent(documentId)}`
: null;
const { data, error, mutate } = useSWRWithDefaults(
@@ -88,118 +50,94 @@ export function useDataSourceViewDocument({
};
}
-async function sendPatchRequest(
- url: string,
- {
- arg,
- }: {
- arg: {
- documentBody: PatchDataSourceWithNameDocumentRequestBody;
- };
- }
-) {
- const res = await fetcherWithBody([url, arg.documentBody, "PATCH"]);
- return res;
-}
-
-function decorateWithInvalidation(
- options: SWRMutationConfiguration | undefined,
- invalidateCacheEntries: () => Promise
-): SWRMutationConfiguration {
- return options
- ? {
- ...options,
- onSuccess: async (data, key, config) => {
- await options.onSuccess?.(data, key, config);
- await invalidateCacheEntries();
- },
- }
- : {
- onSuccess: invalidateCacheEntries,
- };
-}
-
export function useUpdateDataSourceViewDocument(
owner: LightWorkspaceType,
dataSourceView: DataSourceViewType,
- documentName: string,
- options?: SWRMutationConfiguration
+ documentName: string
) {
- // Used only for cache invalidation
- const { mutate: mutateContentNodes } = useDataSourceViewContentNodes({
- owner,
- dataSourceView,
- disabled: true,
- });
+ const { mutateRegardlessOfQueryParams: mutateContentNodes } =
+ useDataSourceViewContentNodes({
+ owner,
+ dataSourceView,
+ disabled: true, // Needed just to create
+ });
// Used only for cache invalidation
const { mutateDocument } = useDataSourceViewDocument({
owner,
dataSourceView,
documentId: documentName,
- disabled: true,
+ disabled: true, // Needed just to create
});
- // Decorate options's onSuccess with cache invalidation
- const invalidateCacheEntries = async () => {
- await Promise.all([mutateContentNodes, mutateDocument]);
+ const sendNotification = useSendNotification();
+
+ const doUpdate = async (body: PatchDataSourceWithNameDocumentRequestBody) => {
+ const patchUrl = `/api/w/${owner.sId}/spaces/${dataSourceView.spaceId}/data_sources/${dataSourceView.dataSource.sId}/documents/${encodeURIComponent(documentName)}`;
+ const res = await fetch(patchUrl, {
+ method: "PATCH",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify(body),
+ });
+ if (!res.ok) {
+ const errorData = await getErrorFromResponse(res);
+ sendNotification({
+ type: "error",
+ title: "Failed to update document",
+ description: `Error: ${errorData.message}`,
+ });
+ return null;
+ } else {
+ void mutateContentNodes();
+ void mutateDocument();
+
+ sendNotification({
+ type: "success",
+ title: "Document updated",
+ description: "Document has been updated",
+ });
+
+ const response: PatchDocumentResponseBody = await res.json();
+ return response.document;
+ }
};
- const decoratedOptions = decorateWithInvalidation(
- options,
- invalidateCacheEntries
- );
-
- const patchUrl = documentName
- ? getUrl({
- method: "PATCH",
- owner,
- dataSourceView,
- documentId: documentName,
- })
- : null;
- return useSWRMutation(patchUrl, sendPatchRequest, decoratedOptions);
-}
-
-async function sendPostRequest(
- url: string,
- {
- arg,
- }: {
- arg: {
- documentBody: PostDataSourceWithNameDocumentRequestBody;
- };
- }
-) {
- const res = await fetcherWithBody([url, arg.documentBody, "POST"]);
- return res;
+ return doUpdate;
}
export function useCreateDataSourceViewDocument(
owner: LightWorkspaceType,
- dataSourceView: DataSourceViewType,
- options?: SWRMutationConfiguration
+ dataSourceView: DataSourceViewType
) {
// Used only for cache invalidation
- const { mutate: mutateContentNodes } = useDataSourceViewContentNodes({
- owner,
- dataSourceView,
- disabled: true,
- });
-
- // Decorate options's onSuccess with cache invalidation
- const invalidateCacheEntries = async () => {
- await mutateContentNodes();
+ const { mutateRegardlessOfQueryParams: mutateContentNodes } =
+ useDataSourceViewContentNodes({
+ owner,
+ dataSourceView,
+ disabled: true,
+ });
+
+ const doCreate = async (body: PostDataSourceWithNameDocumentRequestBody) => {
+ const createUrl = `/api/w/${owner.sId}/spaces/${dataSourceView.spaceId}/data_sources/${dataSourceView.dataSource.sId}/documents`;
+ const res = await fetch(createUrl, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify(body),
+ });
+ if (!res.ok) {
+ const errorData = await getErrorFromResponse(res);
+ console.error("Error creating document", errorData);
+ return null;
+ } else {
+ void mutateContentNodes();
+
+ const response: PostDocumentResponseBody = await res.json();
+ return response.document;
+ }
};
- const decoratedOptions = decorateWithInvalidation(
- options,
- invalidateCacheEntries
- );
- // Note that this url is not used for fetch -> There is no need to invalidate it on practice.
- const createUrl = getUrl({
- method: "POST",
- owner,
- dataSourceView,
- });
- return useSWRMutation(createUrl, sendPostRequest, decoratedOptions);
+ return doCreate;
}
diff --git a/front/lib/swr/data_source_view_tables.ts b/front/lib/swr/data_source_view_tables.ts
new file mode 100644
index 000000000000..e5ddbabe760d
--- /dev/null
+++ b/front/lib/swr/data_source_view_tables.ts
@@ -0,0 +1,181 @@
+import { useSendNotification } from "@dust-tt/sparkle";
+import type { DataSourceViewType, LightWorkspaceType } from "@dust-tt/types";
+import type {
+ PatchDataSourceTableRequestBody,
+ PostDataSourceTableRequestBody,
+} from "@dust-tt/types";
+import { useMemo } from "react";
+import type { Fetcher } from "swr";
+
+import { useDataSourceViewContentNodes } from "@app/lib/swr/data_source_views";
+import {
+ fetcher,
+ getErrorFromResponse,
+ useSWRWithDefaults,
+} from "@app/lib/swr/swr";
+import type { ListTablesResponseBody } from "@app/pages/api/w/[wId]/spaces/[spaceId]/data_source_views/[dsvId]/tables";
+import type { GetDataSourceViewTableResponseBody } from "@app/pages/api/w/[wId]/spaces/[spaceId]/data_source_views/[dsvId]/tables/[tableId]";
+import type { PostTableResponseBody } from "@app/pages/api/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables";
+import type { PatchTableResponseBody } from "@app/pages/api/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/[tableId]";
+
+export function useDataSourceViewTable({
+ dataSourceView,
+ tableId,
+ owner,
+ disabled,
+}: {
+ dataSourceView: DataSourceViewType | null;
+ tableId: string | null;
+ owner: LightWorkspaceType;
+ disabled?: boolean;
+}) {
+ const dataSourceViewTableFetcher: Fetcher =
+ fetcher;
+ const url =
+ dataSourceView && tableId
+ ? `/api/w/${owner.sId}/spaces/${dataSourceView.spaceId}/data_source_views/${dataSourceView.sId}/tables/${encodeURIComponent(tableId)}`
+ : null;
+
+ const { data, error, mutate } = useSWRWithDefaults(
+ url,
+ dataSourceViewTableFetcher,
+ {
+ disabled,
+ }
+ );
+
+ return {
+ table: data?.table,
+ isTableLoading: !disabled && !error && !data,
+ isTableError: error,
+ mutateTable: mutate,
+ };
+}
+
+export function useDataSourceViewTables({
+ dataSourceView,
+ owner,
+}: {
+ dataSourceView: DataSourceViewType | null;
+ owner: LightWorkspaceType;
+}) {
+ const tablesFetcher: Fetcher = fetcher;
+ const disabled = !dataSourceView;
+
+ const url = dataSourceView
+ ? `/api/w/${owner.sId}/spaces/${dataSourceView.spaceId}/data_source_views/${dataSourceView.sId}/tables`
+ : null;
+ const { data, error, mutate } = useSWRWithDefaults(
+ disabled ? null : url,
+ tablesFetcher
+ );
+
+ return {
+ tables: useMemo(() => (data ? data.tables : []), [data]),
+ isTablesLoading: !disabled && !error && !data,
+ isTablesError: error,
+ mutateTables: mutate,
+ };
+}
+export function useUpdateDataSourceViewTable(
+ owner: LightWorkspaceType,
+ dataSourceView: DataSourceViewType,
+ tableId: string
+) {
+ const { mutateRegardlessOfQueryParams: mutateContentNodes } =
+ useDataSourceViewContentNodes({
+ owner,
+ dataSourceView,
+ disabled: true, // Needed just to mutate
+ });
+
+ const { mutateTable } = useDataSourceViewTable({
+ owner,
+ dataSourceView,
+ tableId,
+ disabled: true, // Needed just to mutate
+ });
+
+ const sendNotification = useSendNotification();
+
+ const doUpdate = async (body: PatchDataSourceTableRequestBody) => {
+ const tableUrl = `/api/w/${owner.sId}/spaces/${dataSourceView.spaceId}/data_sources/${dataSourceView.dataSource.sId}/tables/${encodeURIComponent(tableId)}`;
+ const res = await fetch(tableUrl, {
+ method: "PATCH",
+ body: JSON.stringify(body),
+ headers: {
+ "Content-Type": "application/json",
+ },
+ });
+ if (!res.ok) {
+ const errorData = await getErrorFromResponse(res);
+ sendNotification({
+ type: "error",
+ title: "Error creating table",
+ description: `Error: ${errorData.message}`,
+ });
+ console.error("Error updating table", errorData);
+ return null;
+ } else {
+ void mutateContentNodes();
+ void mutateTable();
+
+ sendNotification({
+ type: "success",
+ title: "Table updated",
+ description: "Table has been updated",
+ });
+
+ const response: PatchTableResponseBody = await res.json();
+ return response.table;
+ }
+ };
+
+ return doUpdate;
+}
+
+export function useCreateDataSourceViewTable(
+ owner: LightWorkspaceType,
+ dataSourceView: DataSourceViewType
+) {
+ const { mutateRegardlessOfQueryParams: mutateContentNodes } =
+ useDataSourceViewContentNodes({
+ owner,
+ dataSourceView,
+ disabled: true, // Needed just to mutate
+ });
+ const sendNotification = useSendNotification();
+
+ const doCreate = async (body: PostDataSourceTableRequestBody) => {
+ const tableUrl = `/api/w/${owner.sId}/spaces/${dataSourceView.spaceId}/data_sources/${dataSourceView.dataSource.sId}/tables`;
+ const res = await fetch(tableUrl, {
+ method: "POST",
+ body: JSON.stringify(body),
+ headers: {
+ "Content-Type": "application/json",
+ },
+ });
+ if (!res.ok) {
+ const errorData = await getErrorFromResponse(res);
+ sendNotification({
+ type: "error",
+ title: "Error creating table",
+ description: `Error: ${errorData.message}`,
+ });
+ return null;
+ } else {
+ void mutateContentNodes();
+
+ sendNotification({
+ type: "success",
+ title: "Table created",
+ description: "Table has been created",
+ });
+
+ const response: PostTableResponseBody = await res.json();
+ return response.table;
+ }
+ };
+
+ return doCreate;
+}
diff --git a/front/lib/swr/data_source_views.ts b/front/lib/swr/data_source_views.ts
index 75b42332b692..65b1139d5f25 100644
--- a/front/lib/swr/data_source_views.ts
+++ b/front/lib/swr/data_source_views.ts
@@ -16,7 +16,6 @@ import {
} from "@app/lib/swr/swr";
import type { GetDataSourceViewsResponseBody } from "@app/pages/api/w/[wId]/data_source_views";
import type { GetDataSourceViewContentNodes } from "@app/pages/api/w/[wId]/spaces/[spaceId]/data_source_views/[dsvId]/content-nodes";
-import type { ListTablesResponseBody } from "@app/pages/api/w/[wId]/spaces/[spaceId]/data_source_views/[dsvId]/tables";
import type { GetDataSourceConfigurationResponseBody } from "@app/pages/api/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/configuration";
type DataSourceViewsAndInternalIds = {
@@ -198,28 +197,3 @@ export function useDataSourceViewConnectorConfiguration({
isConfigurationError: error,
};
}
-
-export function useDataSourceViewTables({
- dataSourceView,
- workspaceId,
-}: {
- dataSourceView: DataSourceViewType | null;
- workspaceId: string;
-}) {
- const tablesFetcher: Fetcher = fetcher;
- const disabled = !dataSourceView;
-
- const { data, error, mutate } = useSWRWithDefaults(
- disabled
- ? null
- : `/api/w/${workspaceId}/spaces/${dataSourceView.spaceId}/data_source_views/${dataSourceView.sId}/tables`,
- tablesFetcher
- );
-
- return {
- tables: useMemo(() => (data ? data.tables : []), [data]),
- isTablesLoading: !disabled && !error && !data,
- isTablesError: error,
- mutateTables: mutate,
- };
-}
diff --git a/front/lib/swr/file.ts b/front/lib/swr/file.ts
index 507b06080afb..4db7719b797e 100644
--- a/front/lib/swr/file.ts
+++ b/front/lib/swr/file.ts
@@ -3,6 +3,11 @@ import type { SWRConfiguration } from "swr";
import { useSWRWithDefaults } from "@app/lib/swr/swr";
+export const getFileProcessedUrl = (
+ owner: LightWorkspaceType,
+ fileId: string
+) => `/api/w/${owner.sId}/files/${fileId}?action=view&version=processed`;
+
export function useFileProcessedContent(
owner: LightWorkspaceType,
fileId: string | null,
@@ -17,9 +22,7 @@ export function useFileProcessedContent(
error,
mutate,
} = useSWRWithDefaults(
- isDisabled
- ? null
- : `/api/w/${owner.sId}/files/${fileId}?action=view&version=processed`,
+ isDisabled ? null : getFileProcessedUrl(owner, fileId),
// Stream fetcher -> don't try to parse the stream
// Wait for initial response to trigger swr error handling
async (...args) => {
diff --git a/front/lib/swr/tables.ts b/front/lib/swr/tables.ts
deleted file mode 100644
index 784f89f39a3a..000000000000
--- a/front/lib/swr/tables.ts
+++ /dev/null
@@ -1,34 +0,0 @@
-import type { DataSourceViewType, LightWorkspaceType } from "@dust-tt/types";
-import type { Fetcher } from "swr";
-
-import { fetcher, useSWRWithDefaults } from "@app/lib/swr/swr";
-import type { GetTableResponseBody } from "@app/pages/api/w/[wId]/data_sources/[dsId]/tables/[tId]";
-
-export function useTable({
- owner,
- dataSourceView,
- tableId,
- disabled,
-}: {
- owner: LightWorkspaceType;
- dataSourceView: DataSourceViewType;
- tableId: string | null;
- disabled?: boolean;
-}) {
- const tableFetcher: Fetcher = fetcher;
-
- const endpoint = `/api/w/${owner.sId}/spaces/${dataSourceView.spaceId}/data_source_views/${dataSourceView.sId}/tables/${tableId}`;
- const { data, error, mutate } = useSWRWithDefaults(
- tableId ? endpoint : null,
- tableFetcher,
- {
- disabled,
- }
- );
- return {
- table: data ? data.table : null,
- isTableLoading: !!tableId && !error && !data,
- isTableError: error,
- mutateTable: mutate,
- };
-}
diff --git a/front/package.json b/front/package.json
index 6877764f261b..67f256b195a7 100644
--- a/front/package.json
+++ b/front/package.json
@@ -89,7 +89,6 @@
"next": "^14.2.3",
"next-swagger-doc": "^0.4.0",
"openai": "^4.28.0",
- "pdfjs-dist": "^4.2.67",
"pegjs": "^0.10.0",
"pg": "^8.8.0",
"pg-hstore": "^2.3.4",
@@ -154,7 +153,6 @@
"@types/luxon": "^3.4.2",
"@types/minimist": "^1.2.2",
"@types/node": "^20.12.12",
- "@types/pdfjs-dist": "^2.10.378",
"@types/pegjs": "^0.10.3",
"@types/react": "^18.3.11",
"@types/react-dom": "^18.3.0",
diff --git a/sdks/js/src/types.ts b/sdks/js/src/types.ts
index da65ac450d10..0c124c269e1c 100644
--- a/sdks/js/src/types.ts
+++ b/sdks/js/src/types.ts
@@ -2113,7 +2113,8 @@ const FileTypeUseCaseSchema = FlexibleEnumSchema([
"conversation",
"avatar",
"tool_output",
- "folder",
+ "folder_document",
+ "folder_table",
]);
export const FileTypeSchema = z.object({
diff --git a/types/src/front/api_handlers/public/data_sources.ts b/types/src/front/api_handlers/public/data_sources.ts
index 5667f9d0cb35..ddc68025d8a0 100644
--- a/types/src/front/api_handlers/public/data_sources.ts
+++ b/types/src/front/api_handlers/public/data_sources.ts
@@ -77,6 +77,15 @@ export const PatchDataSourceTableRequestBodySchema = t.intersection([
}),
]);
+export type PatchDataSourceTableRequestBody = t.TypeOf<
+ typeof PatchDataSourceTableRequestBodySchema
+>;
+
+// Post and Patch require the same request body
+export type PostDataSourceTableRequestBody = t.TypeOf<
+ typeof PatchDataSourceTableRequestBodySchema
+>;
+
export const PostDataSourceTableRequestBodySchema = t.intersection([
PatchDataSourceTableRequestBodySchema,
t.type({
diff --git a/types/src/front/files.ts b/types/src/front/files.ts
index d9fd75cdcc42..d4603a8260d3 100644
--- a/types/src/front/files.ts
+++ b/types/src/front/files.ts
@@ -9,7 +9,8 @@ export const FileUploadUrlRequestSchema = t.type({
useCase: t.union([
t.literal("conversation"),
t.literal("avatar"),
- t.literal("folder"),
+ t.literal("folder_document"),
+ t.literal("folder_table"),
]),
useCaseMetadata: t.union([t.undefined, t.type({ conversationId: t.string })]),
});
@@ -64,21 +65,26 @@ export function ensureFileSize(
}
// NOTE: if we add more content types, we need to update the public api package.
+const supportedTabular = {
+ "text/comma-separated-values": [".csv"],
+ "text/csv": [".csv"],
+ "text/tab-separated-values": [".tsv"],
+ "text/tsv": [".tsv"],
+} as const;
// Supported content types for plain text.
const supportedPlainText = {
+ // We support all tabular content types as plain text.
+ ...supportedTabular,
+
"application/msword": [".doc", ".docx"],
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": [
".doc",
".docx",
],
"application/pdf": [".pdf"],
- "text/comma-separated-values": [".csv"],
- "text/csv": [".csv"],
"text/markdown": [".md", ".markdown"],
"text/plain": [".txt"],
- "text/tab-separated-values": [".tsv"],
- "text/tsv": [".tsv"],
} as const;
// Supported content types for images.
@@ -93,6 +99,10 @@ export const supportedPlainTextExtensions = uniq(
Object.values(supportedPlainText).flat()
);
+export const supportedTableExtensions = uniq(
+ Object.values(supportedTabular).flat()
+);
+
export const supportedImageExtensions = uniq(
Object.values(supportedImage).flat()
);
@@ -108,6 +118,9 @@ export const supportedPlainTextContentTypes = Object.keys(
export const supportedImageContentTypes = Object.keys(
supportedImage
) as (keyof typeof supportedImage)[];
+export const supportedTableContentTypes = Object.keys(
+ supportedTabular
+) as (keyof typeof supportedTabular)[];
export const supportedUploadableContentType = [
...supportedPlainTextContentTypes,
@@ -117,6 +130,7 @@ export const supportedUploadableContentType = [
// Infer types from the arrays.
export type PlainTextContentType = keyof typeof supportedPlainText;
export type ImageContentType = keyof typeof supportedImage;
+export type TabularContentType = keyof typeof supportedTabular;
// Union type for all supported content types.
export type SupportedFileContentType = PlainTextContentType | ImageContentType;
@@ -143,11 +157,22 @@ export function isSupportedImageContentType(
return supportedImageContentTypes.includes(contentType as ImageContentType);
}
+export function isSupportedTabularContentType(
+ contentType: string
+): contentType is TabularContentType {
+ return supportedTableContentTypes.includes(contentType as TabularContentType);
+}
+
// Types.
export type FileStatus = "created" | "failed" | "ready";
-export type FileUseCase = "conversation" | "avatar" | "tool_output" | "folder";
+export type FileUseCase =
+ | "conversation"
+ | "avatar"
+ | "tool_output"
+ | "folder_document"
+ | "folder_table";
export type FileUseCaseMetadata = {
conversationId: string;
@@ -179,10 +204,14 @@ export function ensureContentTypeForUseCase(
return isSupportedImageContentType(contentType);
}
- if (useCase === "folder") {
+ if (useCase === "folder_document") {
// Only allow users to upload text documents in folders.
return isSupportedPlainTextContentType(contentType);
}
+ if (useCase === "folder_table") {
+ return isSupportedTabularContentType(contentType);
+ }
+
return false;
}