feat(kw-search): allow passible title and mimetype on front endpoints…

… when upserting tables (#8955) Co-authored-by: Henry Fontanier <[email protected]>
dust-tt · Nov 27, 2024 · 591fe88 · 591fe88
1 parent 66d3d53
commit 591fe88
Show file tree

Hide file tree

Showing 9 changed files with 90 additions and 25 deletions.
diff --git a/front/lib/api/data_sources.ts b/front/lib/api/data_sources.ts
@@ -416,6 +416,8 @@ export async function upsertTable({
   dataSource,
   auth,
   useAppForHeaderDetection,
+  title,
+  mimeType,
 }: {
   tableId?: string | null;
   name: string;
@@ -429,6 +431,8 @@ export async function upsertTable({
   dataSource: DataSourceResource;
   auth: Authenticator;
   useAppForHeaderDetection?: boolean;
+  title?: string;
+  mimeType?: string;
 }) {
   const nonNullTableId = tableId ?? generateRandomModelSId();
   const tableParents: string[] = parents ?? [];
@@ -476,6 +480,8 @@ export async function upsertTable({
         truncate,
         useAppForHeaderDetection: useApp,
         detectedHeaders,
+        title,
+        mimeType,
       },
     });
     if (enqueueRes.isErr()) {
@@ -497,6 +503,8 @@ export async function upsertTable({
     csv: csv ?? null,
     truncate,
     useAppForHeaderDetection: useApp,
+    title,
+    mimeType,
   });
 
   return tableRes;
@@ -644,6 +652,8 @@ export async function handleDataSourceTableCSVUpsert({
         truncate,
         useAppForHeaderDetection,
         detectedHeaders,
+        title: params.title,
+        mimeType: params.mimeType,
       },
     });
     if (enqueueRes.isErr()) {
@@ -673,6 +683,8 @@ export async function handleDataSourceTableCSVUpsert({
     csv: csv ?? null,
     truncate,
     useAppForHeaderDetection,
+    title: params.title,
+    mimeType: params.mimeType,
   });
 
   if (tableRes.isErr()) {

diff --git a/front/lib/api/files/upsert.ts b/front/lib/api/files/upsert.ts
@@ -239,6 +239,8 @@ const upsertTableToDatasource: ProcessingFunction = async ({
     dataSource,
     auth,
     useAppForHeaderDetection: true,
+    title: file.fileName,
+    mimeType: file.contentType,
   });
 
   if (upsertTableRes.isErr()) {

diff --git a/front/lib/api/tables.ts b/front/lib/api/tables.ts
@@ -133,6 +133,8 @@ export async function upsertTableFromCsv({
   truncate,
   useAppForHeaderDetection,
   detectedHeaders,
+  title,
+  mimeType,
 }: {
   auth: Authenticator;
   dataSource: DataSourceResource;
@@ -146,6 +148,8 @@ export async function upsertTableFromCsv({
   truncate: boolean;
   useAppForHeaderDetection: boolean;
   detectedHeaders?: DetectedHeadersType;
+  title?: string;
+  mimeType?: string;
 }): Promise<Result<{ table: CoreAPITable }, TableOperationError>> {
   const csvRowsRes = csv
     ? await rowsFromCsv({
@@ -232,6 +236,8 @@ export async function upsertTableFromCsv({
     timestamp: tableTimestamp,
     tags: tableTags,
     parents: tableParents,
+    title,
+    mimeType,
   });
 
   if (tableRes.isErr()) {

diff --git a/front/lib/upsert_queue.ts b/front/lib/upsert_queue.ts
@@ -44,20 +44,26 @@ const DetectedHeaders = t.type({
   rowIndex: t.number,
 });
 
-export const EnqueueUpsertTable = t.type({
-  workspaceId: t.string,
-  dataSourceId: t.string,
-  tableId: t.string,
-  tableName: t.string,
-  tableDescription: t.string,
-  tableTimestamp: t.union([t.number, t.undefined, t.null]),
-  tableTags: t.union([t.array(t.string), t.undefined, t.null]),
-  tableParents: t.union([t.array(t.string), t.undefined, t.null]),
-  csv: t.union([t.string, t.null]),
-  truncate: t.boolean,
-  useAppForHeaderDetection: t.union([t.boolean, t.undefined, t.null]),
-  detectedHeaders: t.union([DetectedHeaders, t.undefined]),
-});
+export const EnqueueUpsertTable = t.intersection([
+  t.type({
+    workspaceId: t.string,
+    dataSourceId: t.string,
+    tableId: t.string,
+    tableName: t.string,
+    tableDescription: t.string,
+    tableTimestamp: t.union([t.number, t.undefined, t.null]),
+    tableTags: t.union([t.array(t.string), t.undefined, t.null]),
+    tableParents: t.union([t.array(t.string), t.undefined, t.null]),
+    csv: t.union([t.string, t.null]),
+    truncate: t.boolean,
+    useAppForHeaderDetection: t.union([t.boolean, t.undefined, t.null]),
+    detectedHeaders: t.union([DetectedHeaders, t.undefined]),
+  }),
+  t.partial({
+    title: t.string,
+    mimeType: t.string,
+  }),
+]);
 
 type EnqueueUpsertDocumentType = t.TypeOf<typeof EnqueueUpsertDocument>;
 

diff --git a/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/index.ts b/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/index.ts
@@ -252,8 +252,21 @@ async function handler(
         parents,
         remote_database_table_id: remoteDatabaseTableId,
         remote_database_secret_id: remoteDatabaseSecretId,
+        title,
+        mimeType,
       } = r.data;
 
+      // If the request is not from a system key, we enforce that mimeType is not provided.
+      if (!auth.isSystemKey() && mimeType) {
+        return apiError(req, res, {
+          status_code: 400,
+          api_error: {
+            type: "invalid_request_error",
+            message: "Invalid request body: mimeType must not be provided.",
+          },
+        });
+      }
+
       const tableId = maybeTableId || generateRandomModelSId();
 
       const tRes = await coreAPI.getTables({
@@ -302,6 +315,8 @@ async function handler(
         parents: parents || [],
         remoteDatabaseTableId: remoteDatabaseTableId ?? null,
         remoteDatabaseSecretId: remoteDatabaseSecretId ?? null,
+        title,
+        mimeType,
       });
 
       if (upsertRes.isErr()) {

diff --git a/front/temporal/upsert_tables/activities.ts b/front/temporal/upsert_tables/activities.ts
@@ -89,6 +89,8 @@ export async function upsertTableActivity(
     truncate: upsertQueueItem.truncate,
     useAppForHeaderDetection: upsertQueueItem.useAppForHeaderDetection ?? false,
     detectedHeaders: upsertQueueItem.detectedHeaders,
+    title: upsertQueueItem.title,
+    mimeType: upsertQueueItem.mimeType,
   });
 
   if (tableRes.isErr()) {

diff --git a/sdks/js/src/types.ts b/sdks/js/src/types.ts
@@ -1981,6 +1981,8 @@ export const UpsertTableFromCsvRequestSchema = z.intersection(
       truncate: z.boolean(),
       useAppForHeaderDetection: z.boolean().nullable().optional(),
       async: z.boolean().optional(),
+      title: z.string().optional(),
+      mimeType: z.string().optional(),
     })
     .transform((o) => ({
       name: o.name,
@@ -1991,6 +1993,8 @@ export const UpsertTableFromCsvRequestSchema = z.intersection(
       truncate: o.truncate,
       useAppForHeaderDetection: o.useAppForHeaderDetection,
       async: o.async,
+      title: o.title,
+      mimeType: o.mimeType,
     })),
   z.union([
     z.object({ csv: z.string(), tableId: z.undefined() }).transform((o) => ({
@@ -2039,6 +2043,8 @@ export const UpsertDatabaseTableRequestSchema = z.object({
   parents: z.array(z.string()).nullable().optional(),
   remote_database_table_id: z.string().nullable().optional(),
   remote_database_secret_id: z.string().nullable().optional(),
+  title: z.string().optional(),
+  mimeType: z.string().optional(),
 });
 
 const UpsertTableResponseSchema = z.object({

diff --git a/types/src/front/api_handlers/public/data_sources.ts b/types/src/front/api_handlers/public/data_sources.ts
@@ -59,17 +59,23 @@ export type PatchDataSourceWithNameDocumentRequestBody = t.TypeOf<
   typeof PostDataSourceWithNameDocumentRequestBodySchema
 >;
 
-export const PatchDataSourceTableRequestBodySchema = t.type({
-  name: t.string,
-  description: t.string,
-  timestamp: t.union([t.number, t.undefined, t.null]),
-  tags: t.union([t.array(t.string), t.undefined, t.null]),
-  parents: t.union([t.array(t.string), t.undefined, t.null]),
-  truncate: t.boolean,
-  async: t.union([t.boolean, t.undefined]),
-  csv: t.union([t.string, t.undefined]),
-  useAppForHeaderDetection: t.union([t.boolean, t.undefined]),
-});
+export const PatchDataSourceTableRequestBodySchema = t.intersection([
+  t.type({
+    name: t.string,
+    description: t.string,
+    timestamp: t.union([t.number, t.undefined, t.null]),
+    tags: t.union([t.array(t.string), t.undefined, t.null]),
+    parents: t.union([t.array(t.string), t.undefined, t.null]),
+    truncate: t.boolean,
+    async: t.union([t.boolean, t.undefined]),
+    csv: t.union([t.string, t.undefined]),
+    useAppForHeaderDetection: t.union([t.boolean, t.undefined]),
+  }),
+  t.partial({
+    title: t.string,
+    mimeType: t.string,
+  }),
+]);
 
 export const PostDataSourceTableRequestBodySchema = t.intersection([
   PatchDataSourceTableRequestBodySchema,
@@ -97,6 +103,10 @@ export const UpsertTableFromCsvRequestSchema = t.intersection([
       tableId: t.string,
     }),
   ]),
+  t.partial({
+    title: t.string,
+    mimeType: t.string,
+  }),
 ]);
 
 export type UpsertTableFromCsvRequestType = t.TypeOf<

diff --git a/types/src/front/lib/core_api.ts b/types/src/front/lib/core_api.ts
@@ -1087,6 +1087,8 @@ export class CoreAPI {
     parents,
     remoteDatabaseTableId,
     remoteDatabaseSecretId,
+    title,
+    mimeType,
   }: {
     projectId: string;
     dataSourceId: string;
@@ -1098,6 +1100,8 @@ export class CoreAPI {
     parents: string[];
     remoteDatabaseTableId?: string | null;
     remoteDatabaseSecretId?: string | null;
+    title?: string;
+    mimeType?: string;
   }): Promise<CoreAPIResponse<{ table: CoreAPITable }>> {
     const response = await this._fetchWithError(
       `${this._url}/projects/${encodeURIComponent(
@@ -1117,6 +1121,8 @@ export class CoreAPI {
           parents,
           remote_database_table_id: remoteDatabaseTableId ?? null,
           remote_database_secret_id: remoteDatabaseSecretId ?? null,
+          title,
+          mime_type: mimeType,
         }),
       }
     );