From f3acbf4530fbbac73f7c2e7edbeefb11bc2ab5af Mon Sep 17 00:00:00 2001 From: Sebastien Flory Date: Fri, 6 Dec 2024 18:01:00 +0100 Subject: [PATCH] Refactor: simplify mime types and use a common source of truth everywhere (#9207) --- .../data_source/TableUploadOrEditModal.tsx | 4 +- front/hooks/useFileUploaderService.ts | 4 +- front/lib/api/assistant/jit_utils.ts | 5 +- front/lib/api/files/upload.ts | 189 +++++++----------- front/lib/api/files/upsert.ts | 167 ++++++---------- .../resources/content_fragment_resource.ts | 9 +- front/pages/api/v1/w/[wId]/files/index.ts | 4 +- front/pages/api/w/[wId]/files/[fileId].ts | 10 +- front/pages/api/w/[wId]/files/index.ts | 27 ++- types/src/front/content_fragment.ts | 6 - types/src/front/files.ts | 150 ++++---------- 11 files changed, 218 insertions(+), 357 deletions(-) diff --git a/front/components/data_source/TableUploadOrEditModal.tsx b/front/components/data_source/TableUploadOrEditModal.tsx index 74d3cb7375a0..e09ce29ab72e 100644 --- a/front/components/data_source/TableUploadOrEditModal.tsx +++ b/front/components/data_source/TableUploadOrEditModal.tsx @@ -16,8 +16,8 @@ import type { WorkspaceType, } from "@dust-tt/types"; import { - BIG_FILE_SIZE, Err, + isBigFileSize, isSlugified, MAX_FILE_SIZES, maxFileSizeToHumanReadable, @@ -237,7 +237,7 @@ export const TableUploadOrEditModal = ({ name: prev.name.length > 0 ? prev.name : stripTableName(selectedFile.name), })); - setIsBigFile(selectedFile.size > BIG_FILE_SIZE); + setIsBigFile(isBigFileSize(selectedFile.size)); } catch (error) { sendNotification({ type: "error", diff --git a/front/hooks/useFileUploaderService.ts b/front/hooks/useFileUploaderService.ts index d7183e395182..3f8063402858 100644 --- a/front/hooks/useFileUploaderService.ts +++ b/front/hooks/useFileUploaderService.ts @@ -1,7 +1,5 @@ import { useSendNotification } from "@dust-tt/sparkle"; import type { - FileUploadedRequestResponseBody, - FileUploadRequestResponseBody, FileUseCase, FileUseCaseMetadata, LightWorkspaceType, @@ -19,6 +17,8 @@ import { import { useState } from "react"; import { getMimeTypeFromFile } from "@app/lib/file"; +import type { FileUploadRequestResponseBody } from "@app/pages/api/w/[wId]/files"; +import type { FileUploadedRequestResponseBody } from "@app/pages/api/w/[wId]/files/[fileId]"; export interface FileBlob { contentType: SupportedFileContentType; diff --git a/front/lib/api/assistant/jit_utils.ts b/front/lib/api/assistant/jit_utils.ts index e8ab2c1b7d36..8a5b5eec1bbd 100644 --- a/front/lib/api/assistant/jit_utils.ts +++ b/front/lib/api/assistant/jit_utils.ts @@ -90,10 +90,7 @@ function isSearchableContentType( function isListableContentType( contentType: SupportedContentFragmentType ): boolean { - // We allow listing all content-types that are not images. Note that - // `isSupportedPlainTextContentType` is not enough because it is limited to uploadable (as in from - // the conversation) content types which does not cover all non image content types that we - // support in the API such as `dust-application/slack`. + // We allow listing all content-types that are not images. return !isSupportedImageContentType(contentType); } diff --git a/front/lib/api/files/upload.ts b/front/lib/api/files/upload.ts index 8eed3a42b62b..c97a7c71af33 100644 --- a/front/lib/api/files/upload.ts +++ b/front/lib/api/files/upload.ts @@ -4,6 +4,7 @@ import type { SupportedFileContentType, } from "@dust-tt/types"; import { + assertNever, Err, isTextExtractionSupportedContentType, Ok, @@ -28,18 +29,6 @@ import logger from "@app/logger/logger"; const UPLOAD_DELAY_AFTER_CREATION_MS = 1000 * 60 * 1; // 1 minute. -const notSupportedError: ProcessingFunction = async ( - auth: Authenticator, - file: FileResource -) => { - return new Err( - new Error( - "Processing not supported for " + - `content type ${file.contentType} and use case ${file.useCase}` - ) - ); -}; - // Upload to public bucket. const uploadToPublicBucket: ProcessingFunction = async ( @@ -291,121 +280,91 @@ type ProcessingFunction = ( file: FileResource ) => Promise>; -type ProcessingPerUseCase = { - [k in FileUseCase]: ProcessingFunction | undefined; -}; +const getProcessingFunction = ({ + contentType, + useCase, +}: { + contentType: SupportedFileContentType; + useCase: FileUseCase; +}): ProcessingFunction | undefined => { + switch (contentType) { + case "image/jpeg": + case "image/png": + if (useCase === "conversation") { + return resizeAndUploadToFileStorage; + } else if (useCase === "avatar") { + return uploadToPublicBucket; + } + break; + + case "application/msword": + case "application/vnd.openxmlformats-officedocument.wordprocessingml.document": + case "application/pdf": + if (useCase === "conversation" || useCase === "folder_document") { + return extractTextFromFileAndUpload; + } + break; + case "text/plain": + case "text/markdown": + if ( + useCase === "conversation" || + useCase === "folder_document" || + useCase === "tool_output" + ) { + return storeRawText; + } + break; + case "text/vnd.dust.attachment.slack.thread": + if (useCase === "conversation") { + return storeRawText; + } + break; + case "text/comma-separated-values": + case "text/csv": + case "text/tab-separated-values": + case "text/tsv": + if (useCase === "conversation" || useCase === "folder_table") { + // TODO(JIT): after JIT enablement, store raw text here too, the snippet is useless + return extractContentAndSchemaFromDelimitedTextFiles; + } else if (useCase === "folder_document" || useCase === "tool_output") { + return storeRawText; + } + break; + + default: + assertNever(contentType); + } -type ProcessingPerContentType = { - [k in SupportedFileContentType]: ProcessingPerUseCase | undefined; + return undefined; }; -const processingPerContentType: ProcessingPerContentType = { - "application/msword": { - conversation: extractTextFromFileAndUpload, - folder_document: extractTextFromFileAndUpload, - folder_table: notSupportedError, - avatar: notSupportedError, - tool_output: notSupportedError, - }, - "application/vnd.openxmlformats-officedocument.wordprocessingml.document": { - folder_document: extractTextFromFileAndUpload, - folder_table: notSupportedError, - conversation: extractTextFromFileAndUpload, - avatar: notSupportedError, - tool_output: notSupportedError, - }, - "application/pdf": { - folder_document: extractTextFromFileAndUpload, - folder_table: notSupportedError, - conversation: extractTextFromFileAndUpload, - avatar: notSupportedError, - tool_output: notSupportedError, - }, - "image/jpeg": { - conversation: resizeAndUploadToFileStorage, - folder_document: notSupportedError, - folder_table: notSupportedError, - avatar: uploadToPublicBucket, - tool_output: storeRawText, - }, - "image/png": { - conversation: resizeAndUploadToFileStorage, - folder_document: notSupportedError, - folder_table: notSupportedError, - avatar: uploadToPublicBucket, - tool_output: notSupportedError, - }, - "text/comma-separated-values": { - conversation: extractContentAndSchemaFromDelimitedTextFiles, - folder_document: storeRawText, - folder_table: extractContentAndSchemaFromDelimitedTextFiles, - avatar: notSupportedError, - tool_output: storeRawText, - }, - "text/csv": { - conversation: extractContentAndSchemaFromDelimitedTextFiles, - folder_document: storeRawText, - folder_table: extractContentAndSchemaFromDelimitedTextFiles, - avatar: notSupportedError, - tool_output: storeRawText, - }, - "text/markdown": { - conversation: storeRawText, - folder_document: storeRawText, - folder_table: notSupportedError, - avatar: notSupportedError, - tool_output: storeRawText, - }, - "text/plain": { - conversation: storeRawText, - folder_document: storeRawText, - folder_table: notSupportedError, - avatar: notSupportedError, - tool_output: storeRawText, - }, - "text/tab-separated-values": { - conversation: extractContentAndSchemaFromDelimitedTextFiles, - folder_document: storeRawText, - folder_table: extractContentAndSchemaFromDelimitedTextFiles, - avatar: notSupportedError, - tool_output: storeRawText, - }, - "text/tsv": { - conversation: extractContentAndSchemaFromDelimitedTextFiles, - folder_document: storeRawText, - folder_table: extractContentAndSchemaFromDelimitedTextFiles, - avatar: notSupportedError, - tool_output: storeRawText, - }, - "text/vnd.dust.attachment.slack.thread": { - conversation: storeRawText, - folder_document: notSupportedError, - folder_table: notSupportedError, - avatar: notSupportedError, - tool_output: notSupportedError, - }, +export const isUploadSupported = (arg: { + contentType: SupportedFileContentType; + useCase: FileUseCase; +}): boolean => { + const processing = getProcessingFunction(arg); + return !!processing; }; const maybeApplyProcessing: ProcessingFunction = async ( auth: Authenticator, file: FileResource ) => { - const contentTypeProcessing = processingPerContentType[file.contentType]; - if (!contentTypeProcessing) { - return new Ok(undefined); + const processing = getProcessingFunction(file); + if (!processing) { + return new Err( + new Error( + `Processing not supported for content type ${file.contentType} and use case ${file.useCase}` + ) + ); } - const processing = contentTypeProcessing[file.useCase]; - if (processing) { - const res = await processing(auth, file); - if (res.isErr()) { - return res; - } else { - return new Ok(undefined); - } + const res = await processing(auth, file); + if (res.isErr()) { + return res; + } else { + return new Ok(undefined); } - - return new Ok(undefined); }; export async function processAndStoreFile( diff --git a/front/lib/api/files/upsert.ts b/front/lib/api/files/upsert.ts index a93e87c84534..8019c4d64fea 100644 --- a/front/lib/api/files/upsert.ts +++ b/front/lib/api/files/upsert.ts @@ -1,10 +1,14 @@ -import type { FileUseCase, PlainTextContentType, Result } from "@dust-tt/types"; +import type { + FileUseCase, + Result, + SupportedFileContentType, +} from "@dust-tt/types"; import { assertNever, CoreAPI, Err, getSmallWhitelistedModel, - isSupportedPlainTextContentType, + isSupportedDelimitedTextContentType, Ok, removeNulls, slugify, @@ -54,15 +58,6 @@ class MemoryWritable extends Writable { } } -const notSupportedError: ProcessingFunction = async ({ file }) => { - return new Err( - new Error( - "Processing not supported for " + - `content type ${file.contentType} and use case ${file.useCase}` - ) - ); -}; - async function generateSnippet( auth: Authenticator, file: FileResource, @@ -309,85 +304,58 @@ type ProcessingFunction = ({ dataSource: DataSourceResource; }) => Promise>; -type ProcessingPerUseCase = { - [k in FileUseCase]: ProcessingFunction | undefined; -}; +const getProcessingFunction = ({ + contentType, + useCase, +}: { + contentType: SupportedFileContentType; + useCase: FileUseCase; +}): ProcessingFunction | undefined => { + // Use isSupportedDelimitedTextContentType() everywhere to have a common source of truth + if (isSupportedDelimitedTextContentType(contentType)) { + if ( + useCase === "conversation" || + useCase === "tool_output" || + useCase === "folder_table" + ) { + return upsertTableToDatasource; + } else if (useCase === "folder_document") { + return upsertDocumentToDatasource; + } else { + return undefined; + } + } + + switch (contentType) { + case "application/msword": + case "application/vnd.openxmlformats-officedocument.wordprocessingml.document": + case "application/pdf": + case "text/markdown": + case "text/plain": + case "text/vnd.dust.attachment.slack.thread": + if (useCase === "conversation" || useCase === "tool_output") { + return upsertDocumentToDatasource; + } + break; + + case "image/jpeg": + case "image/png": + // We do nothing for images. + break; + + default: + assertNever(contentType); + } -type ProcessingPerContentType = { - [k in PlainTextContentType]: ProcessingPerUseCase | undefined; + return undefined; }; -const processingPerContentType: ProcessingPerContentType = { - "application/msword": { - conversation: upsertDocumentToDatasource, - folder_document: notSupportedError, - folder_table: notSupportedError, - avatar: notSupportedError, - tool_output: notSupportedError, - }, - "application/vnd.openxmlformats-officedocument.wordprocessingml.document": { - conversation: upsertDocumentToDatasource, - folder_document: notSupportedError, - folder_table: notSupportedError, - avatar: notSupportedError, - tool_output: notSupportedError, - }, - "application/pdf": { - conversation: upsertDocumentToDatasource, - folder_document: notSupportedError, - folder_table: notSupportedError, - avatar: notSupportedError, - tool_output: notSupportedError, - }, - "text/comma-separated-values": { - conversation: upsertTableToDatasource, - folder_document: notSupportedError, - folder_table: notSupportedError, - avatar: notSupportedError, - tool_output: upsertTableToDatasource, - }, - "text/csv": { - conversation: upsertTableToDatasource, - folder_document: notSupportedError, - folder_table: notSupportedError, - avatar: notSupportedError, - tool_output: upsertTableToDatasource, - }, - "text/markdown": { - conversation: upsertDocumentToDatasource, - folder_document: notSupportedError, - folder_table: notSupportedError, - avatar: notSupportedError, - tool_output: notSupportedError, - }, - "text/plain": { - conversation: upsertDocumentToDatasource, - folder_document: notSupportedError, - folder_table: notSupportedError, - avatar: notSupportedError, - tool_output: notSupportedError, - }, - "text/tab-separated-values": { - conversation: upsertTableToDatasource, - folder_document: notSupportedError, - folder_table: notSupportedError, - avatar: notSupportedError, - tool_output: notSupportedError, - }, - "text/tsv": { - conversation: upsertTableToDatasource, - folder_document: notSupportedError, - folder_table: notSupportedError, - avatar: notSupportedError, - tool_output: notSupportedError, - }, - "text/vnd.dust.attachment.slack.thread": { - conversation: upsertDocumentToDatasource, - folder_document: notSupportedError, - folder_table: notSupportedError, - avatar: notSupportedError, - tool_output: notSupportedError, - }, +const isUpsertSupported = (arg: { + contentType: SupportedFileContentType; + useCase: FileUseCase; +}): boolean => { + const processing = getProcessingFunction(arg); + return !!processing; }; const maybeApplyProcessing: ProcessingFunction = async ({ @@ -396,14 +364,8 @@ const maybeApplyProcessing: ProcessingFunction = async ({ file, dataSource, }) => { - const contentTypeProcessing = - isSupportedPlainTextContentType(file.contentType) && - processingPerContentType[file.contentType]; - if (!contentTypeProcessing) { - return new Ok(undefined); - } + const processing = getProcessingFunction(file); - const processing = contentTypeProcessing[file.useCase]; if (processing) { const startTime = Date.now(); const res = await processing({ auth, file, content, dataSource }); @@ -463,21 +425,16 @@ export async function processAndUpsertToDataSource( > > { const jitEnabled = await isJITActionsEnabled(auth); - const isJitCompatibleUseCase = - file.useCase === "conversation" || file.useCase === "tool_output"; - const hasJitRequiredMetadata = - isJitCompatibleUseCase && - !!file.useCaseMetadata && - !!file.useCaseMetadata.conversationId; - const isJitSupportedContentType = isSupportedPlainTextContentType( - file.contentType - ); - if (!jitEnabled || !isJitCompatibleUseCase || !isJitSupportedContentType) { + if (!jitEnabled || !isUpsertSupported(file)) { return new Ok(file); } - if (!hasJitRequiredMetadata) { + // Note: this assume that if we don't have useCaseMetadata, the file is fine. + const hasRequiredMetadata = + !!file.useCaseMetadata && !!file.useCaseMetadata.conversationId; + + if (!hasRequiredMetadata) { return new Err({ name: "dust_error", code: "invalid_request_error", diff --git a/front/lib/resources/content_fragment_resource.ts b/front/lib/resources/content_fragment_resource.ts index cfa47537a78e..88f69c1960ee 100644 --- a/front/lib/resources/content_fragment_resource.ts +++ b/front/lib/resources/content_fragment_resource.ts @@ -6,9 +6,10 @@ import type { ModelConfigurationType, ModelId, Result, + SupportedContentFragmentType, WorkspaceType, } from "@dust-tt/types"; -import { Err, isSupportedImageContentFragmentType, Ok } from "@dust-tt/types"; +import { Err, isSupportedImageContentType, Ok } from "@dust-tt/types"; import type { Attributes, CreationAttributes, @@ -359,7 +360,7 @@ export async function renderFromFileId( textBytes, contentFragmentVersion, }: { - contentType: string; + contentType: SupportedContentFragmentType; excludeImages: boolean; fileId: string; forceFullCSVInclude: boolean; @@ -369,7 +370,7 @@ export async function renderFromFileId( contentFragmentVersion: ContentFragmentVersion; } ): Promise> { - if (isSupportedImageContentFragmentType(contentType)) { + if (isSupportedImageContentType(contentType)) { if (excludeImages || !model.supportsVision) { return new Ok({ role: "content_fragment", @@ -449,7 +450,7 @@ export async function renderLightContentFragmentForModel( ): Promise { const { contentType, fileId, title, contentFragmentVersion } = message; - if (fileId && isSupportedImageContentFragmentType(contentType)) { + if (fileId && isSupportedImageContentType(contentType)) { if (excludeImages || !model.supportsVision) { return { role: "content_fragment", diff --git a/front/pages/api/v1/w/[wId]/files/index.ts b/front/pages/api/v1/w/[wId]/files/index.ts index e5d2037486ed..be2e3781379f 100644 --- a/front/pages/api/v1/w/[wId]/files/index.ts +++ b/front/pages/api/v1/w/[wId]/files/index.ts @@ -2,7 +2,6 @@ import type { FileUploadRequestResponseType } from "@dust-tt/client"; import { FileUploadUrlRequestSchema } from "@dust-tt/client"; import type { WithAPIErrorResponse } from "@dust-tt/types"; import { - ensureContentTypeForUseCase, ensureFileSize, isSupportedFileContentType, rateLimiter, @@ -11,6 +10,7 @@ import type { NextApiRequest, NextApiResponse } from "next"; import { fromError } from "zod-validation-error"; import { withPublicAPIAuthentication } from "@app/lib/api/auth_wrappers"; +import { isUploadSupported } from "@app/lib/api/files/upload"; import type { Authenticator } from "@app/lib/auth"; import { FileResource } from "@app/lib/resources/file_resource"; import logger from "@app/logger/logger"; @@ -131,7 +131,7 @@ async function handler( }); } - if (!ensureContentTypeForUseCase(contentType, useCase)) { + if (!isUploadSupported({ contentType, useCase })) { return apiError(req, res, { status_code: 400, api_error: { diff --git a/front/pages/api/w/[wId]/files/[fileId].ts b/front/pages/api/w/[wId]/files/[fileId].ts index 82c1d3be39ba..ebd362b9cf9b 100644 --- a/front/pages/api/w/[wId]/files/[fileId].ts +++ b/front/pages/api/w/[wId]/files/[fileId].ts @@ -1,7 +1,5 @@ -import type { - FileUploadedRequestResponseBody, - WithAPIErrorResponse, -} from "@dust-tt/types"; +import type { FileType } from "@dust-tt/client"; +import type { WithAPIErrorResponse } from "@dust-tt/types"; import type { NextApiRequest, NextApiResponse } from "next"; import { withSessionAuthenticationForWorkspace } from "@app/lib/api/auth_wrappers"; @@ -13,6 +11,10 @@ import { FileResource } from "@app/lib/resources/file_resource"; import logger from "@app/logger/logger"; import { apiError } from "@app/logger/withlogging"; +export interface FileUploadedRequestResponseBody { + file: FileType; +} + export const config = { api: { bodyParser: false, // Disabling Next.js's body parser as formidable has its own. diff --git a/front/pages/api/w/[wId]/files/index.ts b/front/pages/api/w/[wId]/files/index.ts index 65f8e24feec0..723ea6631c48 100644 --- a/front/pages/api/w/[wId]/files/index.ts +++ b/front/pages/api/w/[wId]/files/index.ts @@ -1,24 +1,43 @@ import type { - FileUploadRequestResponseBody, + FileTypeWithUploadUrl, WithAPIErrorResponse, } from "@dust-tt/types"; import { - ensureContentTypeForUseCase, ensureFileSize, - FileUploadUrlRequestSchema, isSupportedFileContentType, rateLimiter, } from "@dust-tt/types"; import { isLeft } from "fp-ts/lib/Either"; +import * as t from "io-ts"; import * as reporter from "io-ts-reporters"; import type { NextApiRequest, NextApiResponse } from "next"; import { withSessionAuthenticationForWorkspace } from "@app/lib/api/auth_wrappers"; +import { isUploadSupported } from "@app/lib/api/files/upload"; import type { Authenticator } from "@app/lib/auth"; import { FileResource } from "@app/lib/resources/file_resource"; import logger from "@app/logger/logger"; import { apiError } from "@app/logger/withlogging"; +// File upload form validation. + +const FileUploadUrlRequestSchema = t.type({ + contentType: t.string, + fileName: t.string, + fileSize: t.number, + useCase: t.union([ + t.literal("conversation"), + t.literal("avatar"), + t.literal("folder_document"), + t.literal("folder_table"), + ]), + useCaseMetadata: t.union([t.undefined, t.type({ conversationId: t.string })]), +}); + +export interface FileUploadRequestResponseBody { + file: FileTypeWithUploadUrl; +} + async function handler( req: NextApiRequest, res: NextApiResponse>, @@ -71,7 +90,7 @@ async function handler( }); } - if (!ensureContentTypeForUseCase(contentType, useCase)) { + if (!isUploadSupported({ contentType, useCase })) { return apiError(req, res, { status_code: 400, api_error: { diff --git a/types/src/front/content_fragment.ts b/types/src/front/content_fragment.ts index 959052be3254..e54fc1a814c2 100644 --- a/types/src/front/content_fragment.ts +++ b/types/src/front/content_fragment.ts @@ -3,8 +3,6 @@ import * as t from "io-ts"; import { ModelId } from "../shared/model_id"; import { MessageType, MessageVisibility } from "./assistant/conversation"; import { - ImageContentType, - supportedImageContentTypes, supportedInlinedContentType, supportedUploadableContentType, } from "./files"; @@ -64,7 +62,3 @@ export function isContentFragmentType( ): arg is ContentFragmentType { return arg.type === "content_fragment"; } - -export function isSupportedImageContentFragmentType(format: string) { - return supportedImageContentTypes.includes(format as ImageContentType); -} diff --git a/types/src/front/files.ts b/types/src/front/files.ts index 1f06a4262882..4335df1a66b9 100644 --- a/types/src/front/files.ts +++ b/types/src/front/files.ts @@ -1,32 +1,32 @@ -import * as t from "io-ts"; - -// File upload form validation. - -export const FileUploadUrlRequestSchema = t.type({ - contentType: t.string, - fileName: t.string, - fileSize: t.number, - useCase: t.union([ - t.literal("conversation"), - t.literal("avatar"), - t.literal("folder_document"), - t.literal("folder_table"), - ]), - useCaseMetadata: t.union([t.undefined, t.type({ conversationId: t.string })]), -}); - -export type FileUploadUrlRequestType = t.TypeOf< - typeof FileUploadUrlRequestSchema ->; - -export interface FileUploadRequestResponseBody { - file: FileTypeWithUploadUrl; -} +// Types. + +export type FileStatus = "created" | "failed" | "ready"; + +export type FileUseCase = + | "conversation" + | "avatar" + | "tool_output" + | "folder_document" + | "folder_table"; + +export type FileUseCaseMetadata = { + conversationId: string; +}; -export interface FileUploadedRequestResponseBody { - file: FileType; +export interface FileType { + contentType: SupportedFileContentType; + downloadUrl?: string; + fileName: string; + fileSize: number; + id: string; + status: FileStatus; + uploadUrl?: string; + publicUrl?: string; + useCase: FileUseCase; } +export type FileTypeWithUploadUrl = FileType & { uploadUrl: string }; + // Define max sizes for each category. export const MAX_FILE_SIZES: Record<"plainText" | "image", number> = { plainText: 30 * 1024 * 1024, // 30MB. @@ -45,8 +45,11 @@ export function maxFileSizeToHumanReadable(size: number) { return `${size / (1024 * 1024)} MB`; } -export const MAX_FILE_LENGTH = 50_000_000; -export const BIG_FILE_SIZE = 5_000_000; +const BIG_FILE_SIZE = 5_000_000; + +export function isBigFileSize(size: number) { + return size > BIG_FILE_SIZE; +} // Function to ensure file size is within max limit for given content type. export function ensureFileSize( @@ -105,32 +108,27 @@ export const supportedPlainTextExtensions = uniq( Object.values(supportedPlainText).flat() ); -export const supportedDelimitedTextExtensions = uniq( - Object.values(supportedDelimitedText).flat() -); - -export const supportedImageExtensions = uniq( - Object.values(supportedImage).flat() -); +const supportedImageExtensions = uniq(Object.values(supportedImage).flat()); export const supportedFileExtensions = uniq([ ...supportedPlainTextExtensions, ...supportedImageExtensions, ]); -export const supportedPlainTextContentTypes = Object.keys( +const supportedPlainTextContentTypes = Object.keys( supportedPlainText ) as (keyof typeof supportedPlainText)[]; -export const supportedImageContentTypes = Object.keys( +const supportedImageContentTypes = Object.keys( supportedImage ) as (keyof typeof supportedImage)[]; -export const supportedDelimitedTextContentTypes = Object.keys( +const supportedDelimitedTextContentTypes = Object.keys( supportedDelimitedText ) as (keyof typeof supportedDelimitedText)[]; -export const supportedRawTextContentTypes = Object.keys( +const supportedRawTextContentTypes = Object.keys( supportedRawText ) as (keyof typeof supportedRawText)[]; +// All the ones listed above export const supportedUploadableContentType = [ ...supportedPlainTextContentTypes, ...supportedImageContentTypes, @@ -141,10 +139,9 @@ export const supportedInlinedContentType = [ ]; // Infer types from the arrays. -export type PlainTextContentType = keyof typeof supportedPlainText; -export type ImageContentType = keyof typeof supportedImage; -export type DelimitedTextContentType = keyof typeof supportedDelimitedText; -export type RawTextContentType = keyof typeof supportedRawText; +type PlainTextContentType = keyof typeof supportedPlainText; +type ImageContentType = keyof typeof supportedImage; +type DelimitedTextContentType = keyof typeof supportedDelimitedText; // Union type for all supported content types. export type SupportedFileContentType = PlainTextContentType | ImageContentType; @@ -157,19 +154,7 @@ export function isSupportedFileContentType( ); } -export type SupportedInlinedContentType = - | DelimitedTextContentType - | RawTextContentType; - -export function isSupportedInlinedContentType( - contentType: string -): contentType is SupportedInlinedContentType { - return supportedInlinedContentType.includes( - contentType as SupportedInlinedContentType - ); -} - -export function isSupportedPlainTextContentType( +function isSupportedPlainTextContentType( contentType: string ): contentType is PlainTextContentType { return supportedPlainTextContentTypes.includes( @@ -204,56 +189,3 @@ export function extensionsForContentType( return []; } - -// Types. - -export type FileStatus = "created" | "failed" | "ready"; - -export type FileUseCase = - | "conversation" - | "avatar" - | "tool_output" - | "folder_document" - | "folder_table"; - -export type FileUseCaseMetadata = { - conversationId: string; -}; - -export interface FileType { - contentType: SupportedFileContentType; - downloadUrl?: string; - fileName: string; - fileSize: number; - id: string; - status: FileStatus; - uploadUrl?: string; - publicUrl?: string; - useCase: FileUseCase; -} - -export type FileTypeWithUploadUrl = FileType & { uploadUrl: string }; - -export function ensureContentTypeForUseCase( - contentType: SupportedFileContentType, - useCase: FileUseCase -) { - if (useCase === "conversation") { - return isSupportedFileContentType(contentType); - } - - if (useCase === "avatar") { - return isSupportedImageContentType(contentType); - } - - if (useCase === "folder_document") { - // Only allow users to upload text documents in folders. - return isSupportedPlainTextContentType(contentType); - } - - if (useCase === "folder_table") { - return isSupportedDelimitedTextContentType(contentType); - } - - return false; -}