From 4b8a9bff9f096028342d3649b61769429c826309 Mon Sep 17 00:00:00 2001 From: Lucas Massemin Date: Thu, 28 Nov 2024 16:34:37 +0100 Subject: [PATCH] static file upload refactoring (#8966) * Can upload multiple documents with fileId * Better notifications * deleted pdfjs code, refactored TableUploader, Split Table and Document for clarity * Got rid of older hooks * Changed folder to document_folder for clarity * Fixed spinner never ending in table modal * More consistent with codebase style * removed unnecessary typing * Populate parent field * Populate name on upload --------- Co-authored-by: Lucas --- .../DataSourceViewDocumentModal.tsx | 1 - ...odal.tsx => DocumentUploadOrEditModal.tsx} | 464 ++---------------- .../data_source/MultipleDocumentsUpload.tsx | 225 +++++---- .../data_source/TableUploadOrEditModal.tsx | 418 ++++++++++++++++ front/components/spaces/ContentActions.tsx | 44 +- front/components/tables/TablePicker.tsx | 5 +- front/hooks/useFileUploaderService.ts | 11 +- front/lib/api/files/upload.ts | 33 +- front/lib/api/files/upsert.ts | 27 +- front/lib/client/handle_file_upload.ts | 118 ----- front/lib/swr/data_source_view_documents.ts | 214 +++----- front/lib/swr/data_source_view_tables.ts | 181 +++++++ front/lib/swr/data_source_views.ts | 26 - front/lib/swr/file.ts | 9 +- front/lib/swr/tables.ts | 34 -- front/package.json | 2 - sdks/js/src/types.ts | 3 +- .../front/api_handlers/public/data_sources.ts | 9 + types/src/front/files.ts | 43 +- 19 files changed, 967 insertions(+), 900 deletions(-) rename front/components/data_source/{DocumentOrTableUploadOrEditModal.tsx => DocumentUploadOrEditModal.tsx} (51%) create mode 100644 front/components/data_source/TableUploadOrEditModal.tsx delete mode 100644 front/lib/client/handle_file_upload.ts create mode 100644 front/lib/swr/data_source_view_tables.ts delete mode 100644 front/lib/swr/tables.ts diff --git a/front/components/DataSourceViewDocumentModal.tsx b/front/components/DataSourceViewDocumentModal.tsx index 1aa19d9be505..8d2e16339530 100644 --- a/front/components/DataSourceViewDocumentModal.tsx +++ b/front/components/DataSourceViewDocumentModal.tsx @@ -25,7 +25,6 @@ export default function DataSourceViewDocumentModal({ dataSourceView, owner, }); - console.log(isDocumentError); const { title, text } = useMemo(() => { if (!document) { diff --git a/front/components/data_source/DocumentOrTableUploadOrEditModal.tsx b/front/components/data_source/DocumentUploadOrEditModal.tsx similarity index 51% rename from front/components/data_source/DocumentOrTableUploadOrEditModal.tsx rename to front/components/data_source/DocumentUploadOrEditModal.tsx index de245f3df418..a4ecd8d0c68b 100644 --- a/front/components/data_source/DocumentOrTableUploadOrEditModal.tsx +++ b/front/components/data_source/DocumentUploadOrEditModal.tsx @@ -2,21 +2,18 @@ import { supportedPlainTextExtensions } from "@dust-tt/client"; import { Button, DocumentPlusIcon, - ExclamationCircleIcon, EyeIcon, EyeSlashIcon, Input, Modal, Page, PlusIcon, - SparklesIcon, Spinner, TextArea, TrashIcon, useSendNotification, } from "@dust-tt/sparkle"; import type { - ContentNodesViewType, CoreAPIDocument, CoreAPILightDocument, DataSourceViewType, @@ -24,27 +21,16 @@ import type { PlanType, WorkspaceType, } from "@dust-tt/types"; -import { - BIG_FILE_SIZE, - Err, - isSlugified, - MAX_FILE_LENGTH, - MAX_FILE_SIZES, - maxFileSizeToHumanReadable, - parseAndStringifyCsv, -} from "@dust-tt/types"; +import { Err } from "@dust-tt/types"; import React, { useCallback, useEffect, useRef, useState } from "react"; import { useFileUploaderService } from "@app/hooks/useFileUploaderService"; -import { handleFileUploadToText } from "@app/lib/client/handle_file_upload"; import { useCreateDataSourceViewDocument, useDataSourceViewDocument, useUpdateDataSourceViewDocument, } from "@app/lib/swr/data_source_view_documents"; import { useFileProcessedContent } from "@app/lib/swr/file"; -import { useTable } from "@app/lib/swr/tables"; -import { useFeatureFlags } from "@app/lib/swr/workspaces"; const MAX_NAME_CHARS = 32; @@ -60,7 +46,13 @@ function isCoreAPIDocumentType( ); } -interface DocumentOrTableUploadOrEditModalProps { +interface Document { + name: string; + text: string; + tags: string[]; + sourceUrl: string; +} +export interface DocumentUploadOrEditModalProps { contentNode?: LightContentNode; dataSourceView: DataSourceViewType; isOpen: boolean; @@ -68,38 +60,17 @@ interface DocumentOrTableUploadOrEditModalProps { owner: WorkspaceType; plan: PlanType; totalNodesCount: number; - viewType: ContentNodesViewType; initialId?: string; } -export function DocumentOrTableUploadOrEditModal( - props: DocumentOrTableUploadOrEditModalProps -) { - const isTable = props.viewType === "tables"; - const initialId = props.contentNode?.internalId; - - return isTable ? ( - - ) : ( - - ); -} - -interface Document { - name: string; - text: string; - tags: string[]; - sourceUrl: string; -} - -const DocumentUploadOrEditModal = ({ +export const DocumentUploadOrEditModal = ({ dataSourceView, isOpen, onClose, owner, plan, initialId, -}: DocumentOrTableUploadOrEditModalProps) => { +}: DocumentUploadOrEditModalProps) => { const sendNotification = useSendNotification(); const fileInputRef = useRef(null); const [documentState, setDocumentState] = useState({ @@ -110,7 +81,7 @@ const DocumentUploadOrEditModal = ({ }); const fileUploaderService = useFileUploaderService({ owner, - useCase: "folder", + useCase: "folder_document", }); const [editionStatus, setEditionStatus] = useState({ @@ -150,84 +121,22 @@ const DocumentUploadOrEditModal = ({ }, shouldRetryOnError: false, }); + const [isUpsertingDocument, setIsUpsertingDocument] = useState(false); - // Side effects of upserting the data source document - const onUpsertSuccess = useCallback(() => { - sendNotification({ - type: "success", - title: `Document successfully ${initialId ? "updated" : "added"}`, - description: `Document ${documentState.name} was successfully ${ - initialId ? "updated" : "added" - }.`, - }); - onClose(true); - setDocumentState({ - name: "", - text: "", - tags: [], - sourceUrl: "", - }); - setEditionStatus({ - content: false, - name: false, - }); - }, [documentState, initialId, onClose, sendNotification]); - - const onUpsertError = useCallback( - (error: unknown) => { - sendNotification({ - type: "error", - title: "Error upserting document", - description: error instanceof Error ? error.message : String(error), - }); - console.error(error); - }, - [sendNotification] - ); - - const onUpsertSettled = useCallback(() => { - setFileId(null); - fileUploaderService.resetUpload(); - }, [fileUploaderService]); - - // Upsert documents to the data source - const patchDocumentMutation = useUpdateDataSourceViewDocument( + const doUpdate = useUpdateDataSourceViewDocument( owner, dataSourceView, - initialId ?? "", - { - onSuccess: () => { - onUpsertSuccess(); - onUpsertSettled(); - }, - onError: (err) => { - onUpsertError(err); - onUpsertSettled(); - }, - } - ); - - const createDocumentMutation = useCreateDataSourceViewDocument( - owner, - dataSourceView, - { - onSuccess: () => { - onUpsertSuccess(); - onUpsertSettled(); - }, - onError: (err) => { - onUpsertError(err); - onUpsertSettled(); - }, - } + initialId ?? "" ); + const doCreate = useCreateDataSourceViewDocument(owner, dataSourceView); const handleDocumentUpload = useCallback( async (document: Document) => { + setIsUpsertingDocument(true); const body = { name: initialId ?? document.name, timestamp: null, - parents: null, + parents: [initialId ?? document.name], section: { prefix: null, content: document.text, sections: [] }, text: null, source_url: document.sourceUrl || undefined, @@ -238,13 +147,35 @@ const DocumentUploadOrEditModal = ({ }; // These mutations do the fetch and mutate, all at once + let upsertRes = null; if (initialId) { - await patchDocumentMutation.trigger({ documentBody: body }); + upsertRes = await doUpdate(body); } else { - await createDocumentMutation.trigger({ documentBody: body }); + upsertRes = await doCreate(body); } + + // Upsert successful, close and reset the modal + if (upsertRes) { + onClose(true); + setDocumentState({ + name: "", + text: "", + tags: [], + sourceUrl: "", + }); + setEditionStatus({ + content: false, + name: false, + }); + } + + // No matter the result, reset the file uploader + setFileId(null); + fileUploaderService.resetUpload(); + setIsUpsertingDocument(false); }, - [createDocumentMutation, patchDocumentMutation, initialId] + + [doUpdate, doCreate, initialId, fileUploaderService, onClose] ); const handleUpload = useCallback(async () => { @@ -290,6 +221,14 @@ const DocumentUploadOrEditModal = ({ // triggers content extraction -> documentState.text update setFileId(fileBlobs[0].fileId); + setDocumentState((prev) => ({ + ...prev, + name: prev.name.length > 0 ? prev.name : selectedFile.name, + sourceUrl: + prev.sourceUrl.length > 0 + ? prev.sourceUrl + : fileBlobs[0].publicUrl ?? "", + })); } catch (error) { sendNotification({ type: "error", @@ -298,7 +237,7 @@ const DocumentUploadOrEditModal = ({ }); } }, - [fileUploaderService, sendNotification] + [fileUploaderService, sendNotification, setDocumentState] ); // Effect: Set the document state when the document is loaded @@ -345,9 +284,7 @@ const DocumentUploadOrEditModal = ({ variant="side-md" title={`${initialId ? "Edit" : "Add"} document`} onSave={handleUpload} - isSaving={ - patchDocumentMutation.isMutating || createDocumentMutation.isMutating - } + isSaving={isUpsertingDocument} > {isDocumentLoading ? (
@@ -525,300 +462,3 @@ const DocumentUploadOrEditModal = ({ ); }; - -interface Table { - name: string; - description: string; - file: File | null; -} - -const TableUploadOrEditModal = ({ - initialId, - dataSourceView, - isOpen, - onClose, - owner, -}: DocumentOrTableUploadOrEditModalProps) => { - const sendNotification = useSendNotification(); - const fileInputRef = useRef(null); - - const [tableState, setTableState] = useState({ - name: "", - description: "", - file: null, - }); - const [editionStatus, setEditionStatus] = useState({ - name: false, - description: false, - }); - const [isUpserting, setIsUpserting] = useState(false); - const [isBigFile, setIsBigFile] = useState(false); - const [isValidTable, setIsValidTable] = useState(false); - const [useAppForHeaderDetection, setUseAppForHeaderDetection] = - useState(false); - - const { table, isTableError, isTableLoading } = useTable({ - owner: owner, - dataSourceView: dataSourceView, - tableId: initialId ?? null, - }); - - const { featureFlags } = useFeatureFlags({ workspaceId: owner.sId }); - - useEffect(() => { - if (!initialId) { - setTableState({ - name: "", - description: "", - file: null, - }); - } else if (table) { - setTableState((prev) => ({ - ...prev, - name: table.name, - description: table.description, - })); - } - }, [initialId, table]); - - useEffect(() => { - const isNameValid = !!tableState.name && isSlugified(tableState.name); - const isContentValid = !!tableState.description; - setIsValidTable(isNameValid && isContentValid && !!tableState.file); - }, [tableState]); - - const handleTableUpload = async (table: Table) => { - setIsUpserting(true); - try { - const fileContent = table.file - ? await handleFileUploadToText(table.file) - : null; - if (fileContent && fileContent.isErr()) { - return new Err(fileContent.error); - } - - const csvContent = fileContent?.value - ? await parseAndStringifyCsv(fileContent.value.content) - : null; - - if (csvContent && csvContent.length > MAX_FILE_LENGTH) { - throw new Error("File too large"); - } - - const base = `/api/w/${owner.sId}/spaces/${dataSourceView.spaceId}/data_sources/${dataSourceView.dataSource.sId}/tables`; - const endpoint = initialId ? `${base}/${initialId}` : base; - - const body = JSON.stringify({ - name: table.name, - description: table.description, - csv: csvContent, - tableId: initialId, - timestamp: null, - tags: [], - parents: [], - truncate: true, - async: false, - useAppForHeaderDetection, - }); - - const res = await fetch(endpoint, { - method: initialId ? "PATCH" : "POST", - headers: { "Content-Type": "application/json" }, - body, - }); - - if (!res.ok) { - throw new Error("Failed to upsert table"); - } - - sendNotification({ - type: "success", - title: `Table successfully ${initialId ? "updated" : "added"}`, - description: `Table ${table.name} was successfully ${initialId ? "updated" : "added"}.`, - }); - } catch (error) { - sendNotification({ - type: "error", - title: "Error upserting table", - description: `An error occurred: ${error instanceof Error ? error.message : String(error)}.`, - }); - } finally { - setIsUpserting(false); - } - }; - - const handleUpload = async () => { - try { - await handleTableUpload(tableState); - onClose(true); - } catch (error) { - console.error(error); - } - }; - - const handleFileChange = async (e: React.ChangeEvent) => { - const selectedFile = e.target.files?.[0]; - if (!selectedFile) { - return; - } - - setIsUpserting(true); - try { - if (selectedFile.size > MAX_FILE_SIZES.plainText) { - sendNotification({ - type: "error", - title: "File too large", - description: `Please upload a file smaller than ${maxFileSizeToHumanReadable(MAX_FILE_SIZES.plainText)}.`, - }); - setIsUpserting(false); - return; - } - - setTableState((prev) => ({ ...prev, file: selectedFile })); - setIsBigFile(selectedFile.size > BIG_FILE_SIZE); - } catch (error) { - sendNotification({ - type: "error", - title: "Error uploading file", - description: error instanceof Error ? error.message : String(error), - }); - } finally { - setIsUpserting(false); - } - }; - - return ( - { - onClose(false); - }} - hasChanged={!isTableError && !isTableLoading && isValidTable} - variant="side-md" - title={`${initialId ? "Edit" : "Add"} table`} - onSave={handleUpload} - isSaving={isUpserting} - > - {isTableLoading ? ( -
- -
- ) : ( - - {isTableError ? ( -
Content cannot be loaded.
- ) : ( -
-
- - { - setEditionStatus((prev) => ({ ...prev, name: true })); - setTableState((prev) => ({ - ...prev, - name: e.target.value, - })); - }} - message={ - editionStatus.name && - (!tableState.name || !isSlugified(tableState.name)) - ? "Invalid name: Must be alphanumeric, max 32 characters and no space." - : null - } - messageStatus="error" - /> -
- -
- -