Merge branch 'main' into code-block-cursor-update

dust-tt · Jan 2, 2025 · 54f01ce · 54f01ce
2 parents 072cc98 + 9b681fc
commit 54f01ce
Show file tree

Hide file tree

Showing 95 changed files with 1,728 additions and 1,036 deletions.
diff --git a/connectors/migrations/20230906_2_slack_fill_parents_field.ts b/connectors/migrations/20230906_2_slack_fill_parents_field.ts
@@ -76,6 +76,7 @@ async function updateParentsFieldForConnector(connector: ConnectorModel) {
           dataSourceConfig: connector,
           documentId: documentIdAndChannel.documentId,
           parents: [documentIdAndChannel.channelId],
+          parentId: null,
         })
       )
     );

diff --git a/connectors/migrations/20230906_3_github_fill_parents_field.ts b/connectors/migrations/20230906_3_github_fill_parents_field.ts
@@ -88,6 +88,7 @@ async function updateDiscussionsParentsFieldForConnector(
             getDiscussionInternalId(document.repoId, document.discussionNumber),
             document.repoId,
           ],
+          parentId: document.repoId,
         });
       })
     );
@@ -118,6 +119,7 @@ async function updateIssuesParentsFieldForConnector(connector: ConnectorModel) {
             getIssueInternalId(document.repoId, document.issueNumber),
             document.repoId,
           ],
+          parentId: document.repoId,
         });
       })
     );

diff --git a/connectors/migrations/20240102_github_add_issues_discussions_parents.ts b/connectors/migrations/20240102_github_add_issues_discussions_parents.ts
@@ -48,6 +48,7 @@ async function updateParents(connector: ConnectorModel) {
             dataSourceConfig: connector,
             documentId,
             parents,
+            parentId: `${d.repoId}-discussions`,
           });
           console.log(`Updated discussion ${documentId} with: ${parents}`);
         } else {
@@ -78,6 +79,7 @@ async function updateParents(connector: ConnectorModel) {
             dataSourceConfig: connector,
             documentId,
             parents,
+            parentId: `${i.repoId}-issues`,
           });
           console.log(`Updated issue ${documentId} with: ${parents}`);
         } else {

diff --git a/connectors/migrations/20240802_table_parents.ts b/connectors/migrations/20240802_table_parents.ts
@@ -49,6 +49,7 @@ async function updateParents({
         tableId,
         parents,
         dataSourceConfig,
+        parentId: parents[1] || null,
       });
     }
   }

diff --git a/connectors/migrations/20240828_microsoft_refill_parents_field.ts b/connectors/migrations/20240828_microsoft_refill_parents_field.ts
@@ -80,6 +80,7 @@ async function updateParentsFieldForConnector(
           dataSourceConfig: dataSourceConfigFromConnector(connector),
           documentId: node.internalId,
           parents,
+          parentId: parents[1] || null,
         });
       },
       { concurrency: 8 }

diff --git a/connectors/migrations/20241030_fix_notion_parents.ts b/connectors/migrations/20241030_fix_notion_parents.ts
@@ -155,6 +155,7 @@ async function updateParentsFieldForConnector(
                 dataSourceConfig: dataSourceConfigFromConnector(connector),
                 documentId,
                 parents,
+                parentId: parents[1] || null,
                 retries: 3,
               });
             }
@@ -163,6 +164,7 @@ async function updateParentsFieldForConnector(
                 dataSourceConfig: dataSourceConfigFromConnector(connector),
                 tableId,
                 parents,
+                parentId: parents[1] || null,
                 retries: 3,
               });
             }

diff --git a/connectors/migrations/20241211_fix_gdrive_parents.ts b/connectors/migrations/20241211_fix_gdrive_parents.ts
@@ -148,6 +148,7 @@ async function migrate({
                   dataSourceConfig,
                   tableId,
                   parents,
+                  parentId: parents[1] || null,
                 });
               }
             }
@@ -210,6 +211,7 @@ async function migrate({
                 dataSourceConfig,
                 tableId,
                 parents,
+                parentId: parents[1] || null,
               });
             }
           }

diff --git a/connectors/migrations/20241212_restore_gdrive_parents.ts b/connectors/migrations/20241212_restore_gdrive_parents.ts
@@ -44,6 +44,7 @@ async function processLogFile(
           dataSourceConfig,
           documentId: documentId,
           parents: previousParents,
+          parentId: previousParents[1] || null,
         });
       }
     }

diff --git a/connectors/migrations/20241216_backfill_confluence_folders.ts b/connectors/migrations/20241216_backfill_confluence_folders.ts
@@ -26,6 +26,7 @@ makeScript({}, async ({ execute }, logger) => {
             dataSourceConfig,
             folderId: makeSpaceInternalId(space.spaceId),
             parents: [makeSpaceInternalId(space.spaceId)],
+            parentId: null,
             title: space.name,
             mimeType: "application/vnd.dust.confluence.space",
           });

diff --git a/connectors/migrations/20241216_backfill_zendesk_folders.ts b/connectors/migrations/20241216_backfill_zendesk_folders.ts
@@ -33,6 +33,7 @@ makeScript({}, async ({ execute }, logger) => {
             dataSourceConfig,
             folderId: brandInternalId,
             parents: [brandInternalId],
+            parentId: null,
             title: brand.name,
             mimeType: "application/vnd.dust.zendesk.brand",
           });
@@ -45,6 +46,7 @@ makeScript({}, async ({ execute }, logger) => {
               helpCenterNode.internalId,
               helpCenterNode.parentInternalId,
             ],
+            parentId: helpCenterNode.parentInternalId,
             title: helpCenterNode.title,
             mimeType: "application/vnd.dust.zendesk.helpcenter",
           });
@@ -54,6 +56,7 @@ makeScript({}, async ({ execute }, logger) => {
             dataSourceConfig,
             folderId: ticketsNode.internalId,
             parents: [ticketsNode.internalId, ticketsNode.parentInternalId],
+            parentId: ticketsNode.parentInternalId,
             title: ticketsNode.title,
             mimeType: "application/vnd.dust.zendesk.tickets",
           });
@@ -81,6 +84,7 @@ makeScript({}, async ({ execute }, logger) => {
             dataSourceConfig: dataSourceConfigFromConnector(connector),
             folderId: parents[0],
             parents,
+            parentId: parents[1],
             title: category.name,
             mimeType: "application/vnd.dust.zendesk.category",
           });

diff --git a/connectors/migrations/20241218_backfill_webcrawler_folders.ts b/connectors/migrations/20241218_backfill_webcrawler_folders.ts
@@ -76,11 +76,13 @@ makeScript(
             execute,
           });
           if (execute) {
+            const parents = getParents(folder);
             const result = await upsertDataSourceFolder({
               dataSourceConfig,
               folderId: folder.internalId,
               timestampMs: folder.updatedAt.getTime(),
-              parents: getParents(folder),
+              parents,
+              parentId: parents[1] || null,
               title: folder.url,
               mimeType: "application/vnd.dust.webcrawler.folder",
             });

diff --git a/connectors/migrations/20241219_backfill_github_folders.ts b/connectors/migrations/20241219_backfill_github_folders.ts
@@ -42,6 +42,7 @@ async function upsertFoldersForConnector(
         dataSourceConfig,
         folderId: repoInternalId,
         parents: [repoInternalId],
+        parentId: null,
         title: repoName,
         mimeType: "application/vnd.dust.github.repository",
       });
@@ -61,6 +62,7 @@ async function upsertFoldersForConnector(
         dataSourceConfig,
         folderId: issuesInternalId,
         parents: [issuesInternalId, repoInternalId],
+        parentId: repoInternalId,
         title: "Issues",
         mimeType: "application/vnd.dust.github.issues",
       });
@@ -76,6 +78,7 @@ async function upsertFoldersForConnector(
         dataSourceConfig,
         folderId: discussionsInternalId,
         parents: [discussionsInternalId, repoInternalId],
+        parentId: repoInternalId,
         title: "Discussions",
         mimeType: "application/vnd.dust.github.discussions",
       });
@@ -96,6 +99,7 @@ async function upsertFoldersForConnector(
           folderId: codeRootInternalId,
           title: "Code",
           parents: [codeRootInternalId, repoInternalId],
+          parentId: repoInternalId,
           mimeType: "application/vnd.dust.github.code.root",
         });
         logger.info(`Upserted code root folder ${codeRootInternalId}`);
@@ -121,6 +125,7 @@ async function upsertFoldersForConnector(
               dataSourceConfig,
               folderId: directory.internalId,
               parents: [directory.internalId, ...dirParents],
+              parentId: dirParents[0] || null,
               title: directory.dirName,
               mimeType: "application/vnd.dust.github.code.directory",
             });

diff --git a/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts b/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts
@@ -34,6 +34,7 @@ async function createFolderNodes(execute: boolean) {
         dataSourceConfig,
         folderId: getTeamsInternalId(connector.id),
         parents: [getTeamsInternalId(connector.id)],
+        parentId: null,
         title: "Conversations",
         mimeType: getDataSourceNodeMimeType("CONVERSATIONS_FOLDER"),
       });
@@ -57,6 +58,7 @@ async function createFolderNodes(execute: boolean) {
             dataSourceConfig,
             folderId: teamInternalId,
             parents: [teamInternalId, getTeamsInternalId(connector.id)],
+            parentId: getTeamsInternalId(connector.id),
             title: team.name,
             mimeType: getDataSourceNodeMimeType("TEAM"),
           });
@@ -95,6 +97,7 @@ async function createFolderNodes(execute: boolean) {
             dataSourceConfig,
             folderId: helpCenterInternalId,
             parents: [helpCenterInternalId],
+            parentId: null,
             title: helpCenter.name,
             mimeType: getDataSourceNodeMimeType("HELP_CENTER"),
           });
@@ -128,6 +131,7 @@ async function createFolderNodes(execute: boolean) {
                 dataSourceConfig,
                 folderId: collectionInternalId,
                 parents: collectionParents,
+                parentId: collectionParents[1] || null,
                 title: collection.name,
                 mimeType: getDataSourceNodeMimeType("COLLECTION"),
               });

diff --git a/connectors/src/api/create_connector.ts b/connectors/src/api/create_connector.ts
@@ -152,9 +152,8 @@ const _createConnectorAPIHandler = async (
     }
 
     if (connectorRes.isErr()) {
-      // Error result means this is an "expected" error, so not
-      // an internal server error.
-      // We return a 4xx status code for expected errors.
+      // Error result means this is an "expected" error, so not an internal server error. We return
+      // a 4xx status code for expected errors.
       switch (connectorRes.error.code) {
         case "INVALID_CONFIGURATION":
           return apiError(req, res, {

diff --git a/connectors/src/api/get_connector_permissions.ts b/connectors/src/api/get_connector_permissions.ts
@@ -1,13 +1,13 @@
+import type { ContentNode } from "@dust-tt/types";
 import type {
   ConnectorPermission,
   WithConnectorsAPIErrorReponse,
 } from "@dust-tt/types";
-import type { ContentNode } from "@dust-tt/types";
+import { assertNever } from "@dust-tt/types";
 import type { Request, Response } from "express";
 
 import { getConnectorManager } from "@connectors/connectors";
 import { augmentContentNodesWithParentIds } from "@connectors/lib/api/content_nodes";
-import { ProviderWorkflowError } from "@connectors/lib/error";
 import logger from "@connectors/logger/logger";
 import { apiError, withLogging } from "@connectors/logger/withlogging";
 import { ConnectorResource } from "@connectors/resources/connector_resource";
@@ -86,25 +86,43 @@ const _getConnectorPermissions = async (
   });
 
   if (pRes.isErr()) {
-    if (
-      pRes.error instanceof ProviderWorkflowError &&
-      pRes.error.type === "rate_limit_error"
-    ) {
-      return apiError(req, res, {
-        status_code: 429,
-        api_error: {
-          type: "connector_rate_limit_error",
-          message: pRes.error.message,
-        },
-      });
+    switch (pRes.error.code) {
+      case "INVALID_PARENT_INTERNAL_ID":
+      case "INVALID_FILTER_PERMISSION":
+        return apiError(req, res, {
+          status_code: 400,
+          api_error: {
+            type: "invalid_request_error",
+            message: pRes.error.message,
+          },
+        });
+      case "CONNECTOR_NOT_FOUND":
+        return apiError(req, res, {
+          status_code: 404,
+          api_error: {
+            type: "connector_not_found",
+            message: pRes.error.message,
+          },
+        });
+      case "EXTERNAL_OAUTH_TOKEN_ERROR":
+        return apiError(req, res, {
+          status_code: 401,
+          api_error: {
+            type: "connector_authorization_error",
+            message: pRes.error.message,
+          },
+        });
+      case "RATE_LIMIT_ERROR":
+        return apiError(req, res, {
+          status_code: 429,
+          api_error: {
+            type: "connector_rate_limit_error",
+            message: pRes.error.message,
+          },
+        });
+      default:
+        assertNever(pRes.error.code);
     }
-    return apiError(req, res, {
-      status_code: 500,
-      api_error: {
-        type: "internal_server_error",
-        message: pRes.error.message,
-      },
-    });
   }
 
   if (includeParents) {