first pass: upserting and deleting folders

connectors/src/connectors/github/temporal/activities.ts

@@ -31,6 +31,8 @@ import {
   renderDocumentTitleAndContent,
   renderMarkdownSection,
   upsertToDatasource,
+  deleteFolderNode,
+  upsertFolderNode,
 } from "@connectors/lib/data_sources";
 import { ExternalOAuthTokenError } from "@connectors/lib/error";
 import {
@@ -668,6 +670,12 @@ export async function githubRepoGarbageCollectActivity(
       repoId: repoId.toString(),
     },
   });
+
+  await deleteFolderNode({
+    dataSourceConfig,
+    folderId: repoId,
+    loggerArgs: logger.bindings(),
+  });
 }
 
 async function deleteIssue(
@@ -846,6 +854,19 @@ async function garbageCollectCodeSync(
         },
       },
     });
+
+    // Also delete data source folders
+    const fq = new PQueue({ concurrency: 8 });
+    directoriesToDelete.forEach((d) =>
+      fq.add(async () => {
+        Context.current().heartbeat();
+        await deleteFolderNode({
+          dataSourceConfig,
+          folderId: d.internalId,
+          loggerArgs: logger.bindings(),
+        });
+      })
+    );
   }
 }
 
@@ -910,6 +931,12 @@ export async function githubCodeSyncActivity({
       },
     });
 
+    await deleteFolderNode({
+      dataSourceConfig,
+      folderId: repoId.toString(),
+      loggerArgs: logger.bindings(),
+    });
+
     return;
   }
 
@@ -941,6 +968,14 @@ export async function githubCodeSyncActivity({
   githubCodeRepository.lastSeenAt = codeSyncStartedAt;
   await githubCodeRepository.save();
 
+  // Add as dataSource folder
+  await upsertFolderNode({
+    dataSourceConfig,
+    folderId: githubCodeRepository.repoId,
+    title: githubCodeRepository.repoName,
+    parents: [githubCodeRepository.repoId],
+  });
+
   logger.info(
     {
       repoId,
@@ -991,6 +1026,12 @@ export async function githubCodeSyncActivity({
         },
       });
 
+      await deleteFolderNode({
+        dataSourceConfig,
+        folderId: repoId.toString(),
+        loggerArgs: logger.bindings(),
+      });
+
       return;
     }
 
@@ -1167,6 +1208,13 @@ export async function githubCodeSyncActivity({
           });
         }
 
+        await upsertFolderNode({
+          dataSourceConfig,
+          folderId: d.internalId,
+          parents: [d.internalId, ...d.parents, repoId.toString()],
+          title: d.dirName,
+        });
+
         // If the parents have updated then the internalId gets updated as well so we should never
         // have an udpate to parentInternalId. We check that this is always the case. If the
         // directory is moved (the parents change) then it will trigger the creation of a new