add concurrent to the migration script

connectors/migrations/20241216_backfill_confluence_folders.ts

@@ -2,10 +2,12 @@ import { makeScript } from "scripts/helpers";
 
 import { makeSpaceInternalId } from "@connectors/connectors/confluence/lib/internal_ids";
 import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config";
+import { concurrentExecutor } from "@connectors/lib/async_utils";
 import { upsertFolderNode } from "@connectors/lib/data_sources";
 import { ConfluenceSpace } from "@connectors/lib/models/confluence";
 import { ConnectorResource } from "@connectors/resources/connector_resource";
 
+const FOLDER_CONCURRENCY = 10;
 makeScript({}, async ({ execute }, logger) => {
   const connectors = await ConnectorResource.listByType("confluence", {});
 
@@ -16,14 +18,18 @@ makeScript({}, async ({ execute }, logger) => {
     });
     const dataSourceConfig = dataSourceConfigFromConnector(connector);
     if (execute) {
-      for (const space of confluenceSpaces) {
-        await upsertFolderNode({
-          dataSourceConfig,
-          folderId: makeSpaceInternalId(space.spaceId),
-          parents: [makeSpaceInternalId(space.spaceId)],
-          title: space.name,
-        });
-      }
+      await concurrentExecutor(
+        confluenceSpaces,
+        async (space) => {
+          await upsertFolderNode({
+            dataSourceConfig,
+            folderId: makeSpaceInternalId(space.spaceId),
+            parents: [makeSpaceInternalId(space.spaceId)],
+            title: space.name,
+          });
+        },
+        { concurrency: FOLDER_CONCURRENCY }
+      );
       logger.info(
         `Upserted ${confluenceSpaces.length} spaces for connector ${connector.id}`
       );