Merge branch 'main' into thomas/permissions-admin

.github/workflows/deploy-core.yml

@@ -3,6 +3,15 @@ name: Deploy Core
 on:
   workflow_dispatch:
     inputs:
+      regions:
+        description: "Regions to deploy to"
+        required: true
+        default: "us-central1"
+        type: choice
+        options:
+          - "us-central1"
+          - "europe-west1"
+          - "all"
       check_deployment_blocked:
         description: "Check #deployment locks or force deploy"
         required: true
@@ -18,17 +27,15 @@ concurrency:
 
 env:
   GCLOUD_PROJECT_ID: ${{ secrets.GCLOUD_PROJECT_ID }}
+  IMAGE_NAME: core
 
 jobs:
-  build-and-deploy:
+  notify-start:
     runs-on: ubuntu-latest
-
-    if: github.ref == 'refs/heads/main'
-
+    outputs:
+      thread_ts: ${{ steps.build_message.outputs.thread_ts }}
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-
+      - uses: actions/checkout@v3
       - name: Get short sha
         id: short_sha
         run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
@@ -52,23 +59,54 @@ jobs:
           channel: ${{ secrets.SLACK_CHANNEL_ID }}
           slack_token: ${{ secrets.SLACK_BOT_TOKEN }}
 
+  create-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - id: set-matrix
+        run: |
+          if [ "${{ github.event.inputs.regions }}" = "all" ]; then
+            echo "matrix=[\"us-central1\",\"europe-west1\"]" >> $GITHUB_OUTPUT
+          else
+            echo "matrix=[\"${{ github.event.inputs.regions }}\"]" >> $GITHUB_OUTPUT
+          fi
+
+  build:
+    needs: [notify-start, create-matrix]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        region: ${{ fromJson(needs.create-matrix.outputs.matrix) }}
+      fail-fast: true
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Get short sha
+        id: short_sha
+        run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
       - name: "Authenticate with Google Cloud"
         uses: "google-github-actions/auth@v1"
         with:
           credentials_json: "${{ secrets.GCLOUD_SA_KEY }}"
 
-      - name: "Set up Cloud SDK"
-        uses: "google-github-actions/setup-gcloud@v1"
-
-      - name: Build the image on Cloud Build
+      - name: Build image for ${{ matrix.region }}
         run: |
           chmod +x ./k8s/cloud-build.sh
           ./k8s/cloud-build.sh \
-            --image-name=core \
+            --image-name=$IMAGE_NAME \
             --dockerfile-path=./Dockerfile \
             --working-dir=./core/ \
-            --region=us-central1
+            --region=${{ matrix.region }}
 
+  deploy:
+    needs: [notify-start, build]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Get short sha
+        id: short_sha
+        run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
       - name: Generate a token
         id: generate-token
         uses: actions/create-github-app-token@v1
@@ -89,10 +127,10 @@ jobs:
               repo: 'dust-infra',
               event_type: 'trigger-component-deploy',
               client_payload: {
-                regions: 'us-central1',
+                regions: '${{ github.event.inputs.regions }}',
                 component: 'core',
                 image_tag: '${{ steps.short_sha.outputs.short_sha }}',
-                slack_thread_ts: "${{ steps.build_message.outputs.thread_ts }}",
+                slack_thread_ts: "${{ needs.notify-start.outputs.thread_ts }}",
                 slack_channel: '${{ secrets.SLACK_CHANNEL_ID }}'
               }
             });
@@ -107,4 +145,4 @@ jobs:
           image_tag: ${{ steps.short_sha.outputs.short_sha }}
           channel: ${{ secrets.SLACK_CHANNEL_ID }}
           slack_token: ${{ secrets.SLACK_BOT_TOKEN }}
-          thread_ts: "${{ steps.build_message.outputs.thread_ts }}"
+          thread_ts: "${{ needs.notify-start.outputs.thread_ts }}"

connectors/migrations/20241218_backfill_slack_folders.ts

@@ -1,7 +1,7 @@
 import { makeScript } from "scripts/helpers";
 import { Op } from "sequelize";
 
-import { internalIdFromSlackChannelId } from "@connectors/connectors/slack/lib/utils";
+import { slackChannelInternalIdFromSlackChannelId } from "@connectors/connectors/slack/lib/utils";
 import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config";
 import { concurrentExecutor } from "@connectors/lib/async_utils";
 import { upsertDataSourceFolder } from "@connectors/lib/data_sources";
@@ -30,7 +30,7 @@ makeScript({}, async ({ execute }, logger) => {
       await concurrentExecutor(
         channels,
         async (channel) => {
-          const internalId = internalIdFromSlackChannelId(
+          const internalId = slackChannelInternalIdFromSlackChannelId(
             channel.slackChannelId
           );
           await upsertDataSourceFolder({

connectors/src/connectors/confluence/temporal/client.ts

@@ -51,6 +51,7 @@ export async function launchConfluenceSyncWorkflow(
   const workflowId = makeConfluenceSyncWorkflowId(connector.id);
 
   const minute = connector.id % 60; // Spread workflows across the hour.
+  const oddOrEvenHour = connector.id % 120 >= 60 ? 1 : 0; // Spread workflows on even or odd hours.
 
   // When the workflow is inactive, we omit passing spaceIds as they are only used to signal modifications within a currently active full sync workflow.
   try {
@@ -70,7 +71,7 @@ export async function launchConfluenceSyncWorkflow(
       memo: {
         connectorId,
       },
-      cronSchedule: `${minute} * * * *`, // Every hour at minute `minute`.
+      cronSchedule: `${minute} ${oddOrEvenHour}/2 * * *`, // Every 2 hours at minute `minute`.
     });
   } catch (err) {
     return new Err(err as Error);

connectors/src/connectors/github/lib/github_api.ts

@@ -850,13 +850,13 @@ export async function processRepository({
       const fileName = basename(file);
 
       const parents = [];
-      for (let i = 0; i < path.length; i++) {
-        const pathInternalId = getCodeDirInternalId(
-          repoId,
-          path.slice(0, i + 1).join("/")
-        );
+      // we order parents bottom to top, so we take paths in the opposite order
+      for (let i = path.length - 1; i >= 0; i--) {
         parents.push({
-          internalId: pathInternalId,
+          internalId: getCodeDirInternalId(
+            repoId,
+            path.slice(0, i + 1).join("/")
+          ),
           dirName: path[i] as string,
           dirPath: path.slice(0, i),
         });
@@ -867,10 +867,7 @@ export async function processRepository({
         `${path.join("/")}/${fileName}`
       );
 
-      const parentInternalId =
-        parents.length === 0
-          ? null
-          : (parents[parents.length - 1]?.internalId as string);
+      const parentInternalId = parents[0]?.internalId ?? null;
 
       // Files
       files.push({
@@ -883,8 +880,7 @@ export async function processRepository({
         sizeBytes: size,
         documentId,
         parentInternalId,
-        /// we reverse the parents here since the convention is bottom to top
-        parents: [documentId, ...parents.map((p) => p.internalId).reverse()],
+        parents: [documentId, ...parents.map((p) => p.internalId)],
         localFilePath: file,
       });
 
@@ -894,7 +890,7 @@ export async function processRepository({
         if (p && !seenDirs[p.internalId]) {
           seenDirs[p.internalId] = true;
 
-          const dirParent = parents[i - 1];
+          const dirParent = parents[i + 1];
           const dirParentInternalId = dirParent ? dirParent.internalId : null;
 
           directories.push({
@@ -906,7 +902,7 @@ export async function processRepository({
             )}`,
             internalId: p.internalId,
             parentInternalId: dirParentInternalId,
-            parents: parents.slice(0, i).map((p) => p.internalId),
+            parents: parents.slice(i).map((p) => p.internalId),
           });
         }
       }

connectors/src/connectors/microsoft/temporal/activities.ts

@@ -107,6 +107,7 @@ export async function getRootNodesToSyncFromResources(
                 connectorId,
                 error,
                 id: resource.internalId,
+                panic: true,
               },
               "Failed to get item"
             );
@@ -669,7 +670,7 @@ export async function syncDeltaForRootNodesInDrive({
         });
 
         if (isMoved) {
-          await updateDescendantsParentsInQdrant({
+          await updateDescendantsParentsInCore({
             dataSourceConfig,
             folder: resource,
             startSyncTs,
@@ -843,7 +844,7 @@ async function isFolderMovedInSameRoot({
   return oldParentId !== newParentId;
 }
 
-async function updateDescendantsParentsInQdrant({
+async function updateDescendantsParentsInCore({
   folder,
   dataSourceConfig,
   startSyncTs,
@@ -855,6 +856,19 @@ async function updateDescendantsParentsInQdrant({
   const children = await folder.fetchChildren();
   const files = children.filter((child) => child.nodeType === "file");
   const folders = children.filter((child) => child.nodeType === "folder");
+
+  await upsertDataSourceFolder({
+    dataSourceConfig,
+    folderId: folder.internalId,
+    parents: await getParents({
+      connectorId: folder.connectorId,
+      internalId: folder.internalId,
+      startSyncTs,
+    }),
+    title: folder.name ?? "",
+    mimeType: "application/vnd.dust.microsoft.folder",
+  });
+
   await concurrentExecutor(
     files,
     async (file) => updateParentsField({ file, dataSourceConfig, startSyncTs }),
@@ -863,7 +877,7 @@ async function updateDescendantsParentsInQdrant({
     }
   );
   for (const childFolder of folders) {
-    await updateDescendantsParentsInQdrant({
+    await updateDescendantsParentsInCore({
       dataSourceConfig,
       folder: childFolder,
       startSyncTs,

connectors/src/connectors/slack/index.ts

@@ -24,8 +24,13 @@ import {
   getSlackClient,
 } from "@connectors/connectors/slack/lib/slack_client";
 import {
-  internalIdFromSlackChannelId,
+  isSlackChannelInternalId,
+  isSlackNonThreadedMessagesInternalId,
+  isSlackThreadInternalId,
   slackChannelIdFromInternalId,
+  slackChannelIdFromSlackNonThreadedMessagesInternalId,
+  slackChannelInternalIdFromSlackChannelId,
+  slackThreadIdentifierFromSlackThreadInternalId,
 } from "@connectors/connectors/slack/lib/utils";
 import { launchSlackSyncWorkflow } from "@connectors/connectors/slack/temporal/client.js";
 import {
@@ -381,7 +386,7 @@ export class SlackConnectorManager extends BaseConnectorManager<SlackConfigurati
 
       const resources: ContentNode[] = slackChannels.map((ch) => ({
         provider: "slack",
-        internalId: internalIdFromSlackChannelId(ch.slackChannelId),
+        internalId: slackChannelInternalIdFromSlackChannelId(ch.slackChannelId),
         parentInternalId: null,
         type: "channel",
         title: `#${ch.slackChannelName}`,
@@ -596,7 +601,7 @@ export class SlackConnectorManager extends BaseConnectorManager<SlackConfigurati
 
     const contentNodes: ContentNode[] = channels.map((ch) => ({
       provider: "slack",
-      internalId: internalIdFromSlackChannelId(ch.slackChannelId),
+      internalId: slackChannelInternalIdFromSlackChannelId(ch.slackChannelId),
       parentInternalId: null,
       type: "channel",
       title: `#${ch.slackChannelName}`,
@@ -617,9 +622,35 @@ export class SlackConnectorManager extends BaseConnectorManager<SlackConfigurati
   }: {
     internalId: string;
   }): Promise<Result<string[], Error>> {
-    // We only ever return permissions at the slack channel level so the parents are always the
-    // internalId itself.
-    return new Ok([internalId]);
+    // If the internal ID is a Slack channel ID, it has no other parent
+    if (isSlackChannelInternalId(internalId)) {
+      return new Ok([internalId]);
+    }
+    // If it is a slack thread, or a slack "non-threaded" message document, it also
+    // needs the channel internal ID as parent
+    else if (isSlackThreadInternalId(internalId)) {
+      const { channelId } =
+        slackThreadIdentifierFromSlackThreadInternalId(internalId);
+      return new Ok([
+        internalId,
+        slackChannelInternalIdFromSlackChannelId(channelId),
+      ]);
+    } else if (isSlackNonThreadedMessagesInternalId(internalId)) {
+      const channelId =
+        slackChannelIdFromSlackNonThreadedMessagesInternalId(internalId);
+      return new Ok([
+        internalId,
+        slackChannelInternalIdFromSlackChannelId(channelId),
+      ]);
+    }
+    // This in theory shouldn't happen
+    else {
+      logger.error(
+        { internalId, panic: true },
+        "Unknown internal ID for Slack connector"
+      );
+      return new Ok([internalId]);
+    }
   }
 
   async setConfigurationKey({