From d931d7b3f96150dabfbf7f60615e6d3f6905eb64 Mon Sep 17 00:00:00 2001
From: Lucas <lucas@dust.tt>
Date: Tue, 10 Dec 2024 15:28:25 +0100
Subject: [PATCH] first pass: upserting and deleting folders

---
 .../connectors/github/temporal/activities.ts  | 48 +++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/connectors/src/connectors/github/temporal/activities.ts b/connectors/src/connectors/github/temporal/activities.ts
index 1de0d13aa8cc..e7a516bb248e 100644
--- a/connectors/src/connectors/github/temporal/activities.ts
+++ b/connectors/src/connectors/github/temporal/activities.ts
@@ -31,6 +31,8 @@ import {
   renderDocumentTitleAndContent,
   renderMarkdownSection,
   upsertToDatasource,
+  deleteFolderNode,
+  upsertFolderNode,
 } from "@connectors/lib/data_sources";
 import { ExternalOAuthTokenError } from "@connectors/lib/error";
 import {
@@ -668,6 +670,12 @@ export async function githubRepoGarbageCollectActivity(
       repoId: repoId.toString(),
     },
   });
+
+  await deleteFolderNode({
+    dataSourceConfig,
+    folderId: repoId,
+    loggerArgs: logger.bindings(),
+  });
 }
 
 async function deleteIssue(
@@ -846,6 +854,19 @@ async function garbageCollectCodeSync(
         },
       },
     });
+
+    // Also delete data source folders
+    const fq = new PQueue({ concurrency: 8 });
+    directoriesToDelete.forEach((d) =>
+      fq.add(async () => {
+        Context.current().heartbeat();
+        await deleteFolderNode({
+          dataSourceConfig,
+          folderId: d.internalId,
+          loggerArgs: logger.bindings(),
+        });
+      })
+    );
   }
 }
 
@@ -910,6 +931,12 @@ export async function githubCodeSyncActivity({
       },
     });
 
+    await deleteFolderNode({
+      dataSourceConfig,
+      folderId: repoId.toString(),
+      loggerArgs: logger.bindings(),
+    });
+
     return;
   }
 
@@ -941,6 +968,14 @@ export async function githubCodeSyncActivity({
   githubCodeRepository.lastSeenAt = codeSyncStartedAt;
   await githubCodeRepository.save();
 
+  // Add as dataSource folder
+  await upsertFolderNode({
+    dataSourceConfig,
+    folderId: githubCodeRepository.repoId,
+    title: githubCodeRepository.repoName,
+    parents: [githubCodeRepository.repoId],
+  });
+
   logger.info(
     {
       repoId,
@@ -991,6 +1026,12 @@ export async function githubCodeSyncActivity({
         },
       });
 
+      await deleteFolderNode({
+        dataSourceConfig,
+        folderId: repoId.toString(),
+        loggerArgs: logger.bindings(),
+      });
+
       return;
     }
 
@@ -1167,6 +1208,13 @@ export async function githubCodeSyncActivity({
           });
         }
 
+        await upsertFolderNode({
+          dataSourceConfig,
+          folderId: d.internalId,
+          parents: [d.internalId, ...d.parents, repoId.toString()],
+          title: d.dirName,
+        });
+
         // If the parents have updated then the internalId gets updated as well so we should never
         // have an udpate to parentInternalId. We check that this is always the case. If the
         // directory is moved (the parents change) then it will trigger the creation of a new