[connectors] Upsert and backfill Confluence spaces as data_source_folders

connectors/migrations/20241216_backfill_confluence_folders.ts

@@ -0,0 +1,43 @@
+import { makeScript } from "scripts/helpers";
+
+import { makeSpaceInternalId } from "@connectors/connectors/confluence/lib/internal_ids";
+import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config";
+import { concurrentExecutor } from "@connectors/lib/async_utils";
+import { upsertFolderNode } from "@connectors/lib/data_sources";
+import { ConfluenceSpace } from "@connectors/lib/models/confluence";
+import { ConnectorResource } from "@connectors/resources/connector_resource";
+
+const FOLDER_CONCURRENCY = 10;
+
+makeScript({}, async ({ execute }, logger) => {
+  const connectors = await ConnectorResource.listByType("confluence", {});
+
+  for (const connector of connectors) {
+    const confluenceSpaces = await ConfluenceSpace.findAll({
+      attributes: ["spaceId", "name"],
+      where: { connectorId: connector.id },
+    });
+    const dataSourceConfig = dataSourceConfigFromConnector(connector);
+    if (execute) {
+      await concurrentExecutor(
+        confluenceSpaces,
+        async (space) => {
+          await upsertFolderNode({
+            dataSourceConfig,
+            folderId: makeSpaceInternalId(space.spaceId),
+            parents: [makeSpaceInternalId(space.spaceId)],
+            title: space.name,
+          });
+        },
+        { concurrency: FOLDER_CONCURRENCY }
+      );
+      logger.info(
+        `Upserted ${confluenceSpaces.length} spaces for connector ${connector.id}`
+      );
+    } else {
+      logger.info(
+        `Found ${confluenceSpaces.length} spaces for connector ${connector.id}`
+      );
+    }
+  }
+});

connectors/src/connectors/confluence/index.ts

@@ -44,7 +44,12 @@ import {
   BaseConnectorManager,
   ConnectorManagerError,
 } from "@connectors/connectors/interface";
+import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config";
 import { concurrentExecutor } from "@connectors/lib/async_utils";
+import {
+  deleteFolderNode,
+  upsertFolderNode,
+} from "@connectors/lib/data_sources";
 import {
   ConfluenceConfiguration,
   ConfluencePage,
@@ -322,6 +327,7 @@ export class ConfluenceConnectorManager extends BaseConnectorManager<null> {
         new Error(`Connector not found with id ${this.connectorId}`)
       );
     }
+    const dataSourceConfig = dataSourceConfigFromConnector(connector);
 
     let spaces: ConfluenceSpaceType[] = [];
     // Fetch Confluence spaces only if the intention is to add new spaces to sync.
@@ -349,6 +355,8 @@ export class ConfluenceConnectorManager extends BaseConnectorManager<null> {
           },
         });
 
+        await deleteFolderNode({ dataSourceConfig, folderId: internalId });
+
         removedSpaceIds.push(confluenceId);
       } else if (permission === "read") {
         const confluenceSpace = spaces.find((s) => s.id === confluenceId);
@@ -360,6 +368,13 @@ export class ConfluenceConnectorManager extends BaseConnectorManager<null> {
           urlSuffix: confluenceSpace?._links.webui,
         });
 
+        await upsertFolderNode({
+          dataSourceConfig,
+          folderId: internalId,
+          parents: [internalId],
+          title: confluenceSpace?.name ?? confluenceId,
+        });
+
         addedSpaceIds.push(confluenceId);
       } else {
         return new Err(