[connectors] Add title and mimetypes (#9097)

dust-tt · Dec 4, 2024 · 516c9e7 · 516c9e7
1 parent 92f8c33
commit 516c9e7
Show file tree

Hide file tree

Showing 19 changed files with 73 additions and 15 deletions.
diff --git a/connectors/src/connectors/confluence/temporal/activities.ts b/connectors/src/connectors/confluence/temporal/activities.ts
@@ -346,6 +346,8 @@ export async function confluenceCheckAndUpsertPageActivity({
       upsertContext: {
         sync_type: isBatchSync ? "batch" : "incremental",
       },
+      title: page.title,
+      mimeType: "application/vnd.dust.confluence-page",
       async: true,
     });
   }

diff --git a/connectors/src/connectors/github/temporal/activities.ts b/connectors/src/connectors/github/temporal/activities.ts
@@ -296,6 +296,8 @@ export async function githubUpsertIssueActivity(
     upsertContext: {
       sync_type: isBatchSync ? "batch" : "incremental",
     },
+    title: issue.title,
+    mimeType: "application/vnd.dust.github-issue",
     async: true,
   });
 
@@ -480,6 +482,8 @@ export async function githubUpsertDiscussionActivity(
     upsertContext: {
       sync_type: isBatchSync ? "batch" : "incremental",
     },
+    title: discussion.title,
+    mimeType: "application/vnd.dust.github-discussion",
     async: true,
   });
 
@@ -1093,6 +1097,8 @@ export async function githubCodeSyncActivity({
             upsertContext: {
               sync_type: isBatchSync ? "batch" : "incremental",
             },
+            title: f.fileName,
+            mimeType: "application/vnd.dust.github-code-file",
             async: true,
           });
 

diff --git a/connectors/src/connectors/google_drive/temporal/file.ts b/connectors/src/connectors/google_drive/temporal/file.ts
@@ -487,6 +487,8 @@ async function upsertGdriveDocument(
       upsertContext: {
         sync_type: isBatchSync ? "batch" : "incremental",
       },
+      title: file.name,
+      mimeType: file.mimeType,
       async: true,
     });
     return file.updatedAtMs;

diff --git a/connectors/src/connectors/intercom/temporal/sync_conversation.ts b/connectors/src/connectors/intercom/temporal/sync_conversation.ts
@@ -320,6 +320,8 @@ export async function syncConversation({
     upsertContext: {
       sync_type: syncType,
     },
+    title: convoTitle,
+    mimeType: "application/vnd.dust.intercom-conversation",
     async: true,
   });
 }
diff --git a/connectors/src/connectors/intercom/temporal/sync_help_center.ts b/connectors/src/connectors/intercom/temporal/sync_help_center.ts
@@ -389,6 +389,8 @@ export async function upsertArticle({
       upsertContext: {
         sync_type: "batch",
       },
+      title: article.title,
+      mimeType: "application/vnd.dust.intercom-article",
       async: true,
     });
     await articleOnDb.update({

diff --git a/connectors/src/connectors/microsoft/temporal/file.ts b/connectors/src/connectors/microsoft/temporal/file.ts
@@ -308,6 +308,8 @@ export async function syncOneFile({
           upsertContext: {
             sync_type: isBatchSync ? "batch" : "incremental",
           },
+          title: file.name ?? "",
+          mimeType: file.file.mimeType ?? "application/octet-stream",
           async: true,
         });
 

diff --git a/connectors/src/connectors/notion/temporal/activities.ts b/connectors/src/connectors/notion/temporal/activities.ts
@@ -2032,6 +2032,8 @@ export async function renderAndUpsertPageFromCache({
       upsertContext: {
         sync_type: isFullSync ? "batch" : "incremental",
       },
+      title: title ?? "",
+      mimeType: "application/vnd.notion.page",
       async: true,
     });
   }
@@ -2525,7 +2527,7 @@ export async function upsertDatabaseStructuredDataFromCache({
         truncate: true,
         parents,
         title: dbModel.title ?? "Untitled Notion Database",
-        mimeType: "notion/database",
+        mimeType: "application/vnd.notion.database",
       }),
     localLogger
   );
@@ -2579,6 +2581,8 @@ export async function upsertDatabaseStructuredDataFromCache({
         upsertContext: {
           sync_type: "batch",
         },
+        title: databaseName,
+        mimeType: "application/vnd.notion.database",
         async: true,
       });
     } else {

diff --git a/connectors/src/connectors/slack/temporal/activities.ts b/connectors/src/connectors/slack/temporal/activities.ts
@@ -592,6 +592,8 @@ export async function syncNonThreaded(
     upsertContext: {
       sync_type: isBatchSync ? "batch" : "incremental",
     },
+    title: tags.find((t) => t.startsWith("title:"))?.split(":")[1] ?? "",
+    mimeType: "text/vnd.dust.slack-thread",
     async: true,
   });
 }
@@ -784,6 +786,8 @@ export async function syncThread(
     upsertContext: {
       sync_type: isBatchSync ? "batch" : "incremental",
     },
+    title: tags.find((t) => t.startsWith("title:"))?.split(":")[1] ?? "",
+    mimeType: "text/vnd.dust.slack-thread",
     async: true,
   });
 }

diff --git a/connectors/src/connectors/webcrawler/temporal/activities.ts b/connectors/src/connectors/webcrawler/temporal/activities.ts
@@ -332,6 +332,8 @@ export async function crawlWebsiteByConnectorId(connectorId: ModelId) {
               upsertContext: {
                 sync_type: "batch",
               },
+              title: pageTitle,
+              mimeType: "text/html",
               async: true,
             });
           } else {

diff --git a/connectors/src/connectors/zendesk/lib/sync_article.ts b/connectors/src/connectors/zendesk/lib/sync_article.ts
@@ -165,6 +165,8 @@ export async function syncArticle({
       parents: articleInDb.getParentInternalIds(connectorId),
       loggerArgs: { ...loggerArgs, articleId: article.id },
       upsertContext: { sync_type: "batch" },
+      title: article.title,
+      mimeType: "application/vnd.dust.zendesk-article",
       async: true,
     });
     await articleInDb.update({ lastUpsertedTs: new Date(currentSyncDateMs) });

diff --git a/connectors/src/connectors/zendesk/lib/sync_ticket.ts b/connectors/src/connectors/zendesk/lib/sync_ticket.ts
@@ -224,6 +224,8 @@ ${comments
       parents: ticketInDb.getParentInternalIds(connectorId),
       loggerArgs: { ...loggerArgs, ticketId: ticket.id },
       upsertContext: { sync_type: "batch" },
+      title: ticket.subject,
+      mimeType: "application/vnd.dust.zendesk-ticket",
       async: true,
     });
     await ticketInDb.update({ lastUpsertedTs: new Date(currentSyncDateMs) });

diff --git a/connectors/src/lib/data_sources.ts b/connectors/src/lib/data_sources.ts
@@ -1,3 +1,7 @@
+import type {
+  UpsertDatabaseTableRequestType,
+  UpsertTableFromCsvRequestType,
+} from "@dust-tt/client";
 import { DustAPI } from "@dust-tt/client";
 import type {
   CoreAPIDataSourceDocumentSection,
@@ -64,6 +68,8 @@ type UpsertToDataSourceParams = {
   parents: string[];
   loggerArgs?: Record<string, string | number>;
   upsertContext: UpsertContext;
+  title: string;
+  mimeType: string;
   async: boolean;
 };
 
@@ -79,6 +85,8 @@ async function _upsertToDatasource({
   parents,
   loggerArgs = {},
   upsertContext,
+  title,
+  mimeType,
   async,
 }: UpsertToDataSourceParams) {
   return tracer.trace(
@@ -130,6 +138,8 @@ async function _upsertToDatasource({
         section: documentContent,
         source_url: documentUrl,
         timestamp,
+        title,
+        mime_type: mimeType,
         tags: tags?.map((tag) => tag.substring(0, 512)),
         parents,
         light_document_output: true,
@@ -598,15 +608,15 @@ export async function upsertTableFromConnectors({
   const endpoint =
     `${DUST_FRONT_API}/api/v1/w/${dataSourceConfig.workspaceId}` +
     `/data_sources/${dataSourceConfig.dataSourceId}/tables`;
-  const dustRequestPayload = {
+  const dustRequestPayload: UpsertDatabaseTableRequestType = {
     name: tableName,
     parents,
     description: tableDescription,
     table_id: tableId,
     remote_database_table_id: remoteDatabaseTableId,
     remote_database_secret_id: remoteDatabaseSecretId,
     title,
-    mimeType,
+    mime_type: mimeType,
   };
   const dustRequestConfig: AxiosRequestConfig = {
     headers: {
@@ -755,7 +765,7 @@ export async function upsertTableFromCsv({
   const endpoint =
     `${DUST_FRONT_API}/api/v1/w/${dataSourceConfig.workspaceId}` +
     `/data_sources/${dataSourceConfig.dataSourceId}/tables/csv`;
-  const dustRequestPayload = {
+  const dustRequestPayload: UpsertTableFromCsvRequestType = {
     name: tableName,
     parents,
     description: tableDescription,
@@ -766,6 +776,8 @@ export async function upsertTableFromCsv({
     useAppForHeaderDetection,
     title,
     mimeType,
+    timestamp: null,
+    tags: null,
   };
   const dustRequestConfig: AxiosRequestConfig = {
     headers: {
@@ -803,7 +815,7 @@ export async function upsertTableFromCsv({
           error: sanitizedError,
           payload: {
             ...dustRequestPayload,
-            csv: dustRequestPayload.csv.substring(0, 100),
+            csv: tableCsv.substring(0, 100),
           },
         },
         "Axios error uploading table to Dust."
@@ -814,7 +826,7 @@ export async function upsertTableFromCsv({
           error: e.message,
           payload: {
             ...dustRequestPayload,
-            csv: dustRequestPayload.csv.substring(0, 100),
+            csv: tableCsv.substring(0, 100),
           },
         },
         "Error uploading table to Dust."

diff --git a/front/components/data_source/DocumentUploadOrEditModal.tsx b/front/components/data_source/DocumentUploadOrEditModal.tsx
@@ -138,7 +138,7 @@ export const DocumentUploadOrEditModal = ({
       const body = {
         name: initialId ?? document.name,
         title: initialId ?? document.name,
-        mime_type: document.mimeType,
+        mime_type: document.mimeType ?? undefined,
         timestamp: null,
         parents: [initialId ?? document.name],
         section: { prefix: null, content: document.text, sections: [] },

diff --git a/front/components/data_source/MultipleDocumentsUpload.tsx b/front/components/data_source/MultipleDocumentsUpload.tsx
@@ -141,6 +141,8 @@ export const MultipleDocumentsUpload = ({
         // Create the document
         const body = {
           name: blob.filename,
+          title: blob.filename,
+          mime_type: blob.contentType ?? undefined,
           timestamp: null,
           parents: [blob.filename],
           section: {

diff --git a/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/csv.ts b/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/csv.ts
@@ -117,7 +117,6 @@ async function handler(
         auth,
         params: {
           ...r.data,
-          mimeType: r.data.mimeType,
           title: r.data.title ?? r.data.name,
         },
         dataSource,

diff --git a/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/index.ts b/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/index.ts
@@ -114,13 +114,16 @@ import { apiError } from "@app/logger/withlogging";
  *                 items:
  *                   type: string
  *                 description: Parent tables of this table
+ *               mime_type:
+ *                 type: string
+ *                 description: Mime type of the table
  *     responses:
  *       200:
  *         description: The table
  *         content:
  *           application/json:
  *             schema:
- *               $ref: '#/components/schemas/Datasource'
+ *               $ref: '#/components/schemas/Table'
  *       400:
  *         description: Invalid request
  *       405:
@@ -264,7 +267,7 @@ async function handler(
       let title: string;
       if (auth.isSystemKey()) {
         // If the request is from a system key, the request must provide both title and mimeType.
-        if (!r.data.mimeType) {
+        if (!r.data.mime_type) {
           return apiError(req, res, {
             status_code: 400,
             api_error: {
@@ -283,11 +286,11 @@ async function handler(
           });
         }
 
-        mimeType = r.data.mimeType;
+        mimeType = r.data.mime_type;
         title = r.data.title;
       } else {
         // If the request is from a regular API key, the request must not provide mimeType.
-        if (r.data.mimeType) {
+        if (r.data.mime_type) {
           return apiError(req, res, {
             status_code: 400,
             api_error: {

diff --git a/front/public/swagger.json b/front/public/swagger.json
@@ -3179,6 +3179,10 @@
                       "type": "string"
                     },
                     "description": "Parent tables of this table"
+                  },
+                  "mime_type": {
+                    "type": "string",
+                    "description": "Mime type of the table"
                   }
                 }
               }
@@ -3191,7 +3195,7 @@
             "content": {
               "application/json": {
                 "schema": {
-                  "$ref": "#/components/schemas/Datasource"
+                  "$ref": "#/components/schemas/Table"
                 }
               }
             }

diff --git a/sdks/js/src/types.ts b/sdks/js/src/types.ts
@@ -2021,6 +2021,10 @@ export const UpsertTableFromCsvRequestSchema = z.intersection(
   ])
 );
 
+export type UpsertTableFromCsvRequestType = z.infer<
+  typeof UpsertTableFromCsvRequestSchema
+>;
+
 const PostTableCSVAsyncResponseSchema = z.object({
   table: z.object({
     table_id: z.string(),
@@ -2052,9 +2056,13 @@ export const UpsertDatabaseTableRequestSchema = z.object({
   remote_database_table_id: z.string().nullable().optional(),
   remote_database_secret_id: z.string().nullable().optional(),
   title: z.string().optional(),
-  mimeType: z.string().optional(),
+  mime_type: z.string().optional(),
 });
 
+export type UpsertDatabaseTableRequestType = z.infer<
+  typeof UpsertDatabaseTableRequestSchema
+>;
+
 const UpsertTableResponseSchema = z.object({
   table: CoreAPITablePublicSchema,
 });

diff --git a/types/src/front/api_handlers/public/data_sources.ts b/types/src/front/api_handlers/public/data_sources.ts
@@ -41,7 +41,7 @@ export const PostDataSourceDocumentRequestBodySchema = t.intersection([
   }),
   t.partial({
     title: t.string,
-    mimeType: t.string,
+    mime_type: t.string,
   }),
 ]);