Merge branch 'main' into thomas/parent-id

dust-tt · Dec 11, 2024 · 9582189 · 9582189
2 parents 401bd03 + 93cfac7
commit 9582189
Show file tree

Hide file tree

Showing 49 changed files with 1,597 additions and 69 deletions.
diff --git a/.github/workflows/deploy-alerting-temporal.yml b/.github/workflows/deploy-alerting-temporal.yml
@@ -44,7 +44,8 @@ jobs:
             --image-name=alerting-temporal \
             --dockerfile-path=./Dockerfile \
             --working-dir=./alerting/temporal/ \
-            --dust-client-facing-url=https://dust.tt
+            --dust-client-facing-url=https://dust.tt \ 
+            --region=us-central1
 
       - name: Generate a token
         id: generate-token

diff --git a/.github/workflows/deploy-connectors.yml b/.github/workflows/deploy-connectors.yml
@@ -67,7 +67,8 @@ jobs:
             --image-name=connectors \
             --dockerfile-path=./connectors/Dockerfile \
             --working-dir=./ \
-            --dust-client-facing-url=https://dust.tt
+            --dust-client-facing-url=https://dust.tt \ 
+            --region=us-central1
 
       - name: Generate a token
         id: generate-token

diff --git a/.github/workflows/deploy-core.yml b/.github/workflows/deploy-core.yml
@@ -67,7 +67,8 @@ jobs:
             --image-name=core \
             --dockerfile-path=./Dockerfile \
             --working-dir=./core/ \
-            --dust-client-facing-url=https://dust.tt
+            --dust-client-facing-url=https://dust.tt \ 
+            --region=us-central1
 
       - name: Generate a token
         id: generate-token

diff --git a/.github/workflows/deploy-dante.yml b/.github/workflows/deploy-dante.yml
@@ -54,7 +54,8 @@ jobs:
             --dockerfile-path=./dante.Dockerfile \
             --working-dir=./ \
             --gcloud-ignore-file=../.gcloudignore-dante \
-            --dust-client-facing-url=https://dust.tt
+            --dust-client-facing-url=https://dust.tt \ 
+            --region=us-central1
           cd ..
 
       - name: Generate a token

diff --git a/.github/workflows/deploy-front-edge.yml b/.github/workflows/deploy-front-edge.yml
@@ -3,6 +3,15 @@ name: Deploy Front Edge
 on:
   workflow_dispatch:
     inputs:
+      regions:
+        description: "Regions to deploy to"
+        required: true
+        default: "us-central1"
+        type: choice
+        options:
+          - "us-central1"
+          - "europe-west1"
+          - "all"
       check_deployment_blocked:
         description: "Check #deployment locks or force deploy"
         required: true
@@ -21,13 +30,12 @@ env:
   IMAGE_NAME: front-edge
 
 jobs:
-  build-and-deploy:
+  notify-start:
     runs-on: ubuntu-latest
-
+    outputs:
+      thread_ts: ${{ steps.build_message.outputs.thread_ts }}
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-
+      - uses: actions/checkout@v3
       - name: Get short sha
         id: short_sha
         run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
@@ -51,20 +59,55 @@ jobs:
           channel: ${{ secrets.SLACK_CHANNEL_ID }}
           slack_token: ${{ secrets.SLACK_BOT_TOKEN }}
 
+  create-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - id: set-matrix
+        run: |
+          if [ "${{ github.event.inputs.regions }}" = "all" ]; then
+            echo "matrix=[\"us-central1\",\"europe-west1\"]" >> $GITHUB_OUTPUT
+          else
+            echo "matrix=[\"${{ github.event.inputs.regions }}\"]" >> $GITHUB_OUTPUT
+          fi
+
+  build:
+    needs: [notify-start, create-matrix]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        region: ${{ fromJson(needs.create-matrix.outputs.matrix) }}
+      fail-fast: true
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Get short sha
+        id: short_sha
+        run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
       - name: "Authenticate with Google Cloud"
         uses: "google-github-actions/auth@v1"
         with:
           credentials_json: "${{ secrets.GCLOUD_SA_KEY }}"
 
-      - name: Build the image on Cloud Build
+      - name: Build image for ${{ matrix.region }}
         run: |
           chmod +x ./k8s/cloud-build.sh
           ./k8s/cloud-build.sh \
             --image-name=$IMAGE_NAME \
             --dockerfile-path=./front/Dockerfile \
             --working-dir=./ \
-            --dust-client-facing-url=https://front-edge.dust.tt
+            --dust-client-facing-url=https://front-edge.dust.tt \
+            --region=${{ matrix.region }}
 
+  deploy:
+    needs: [notify-start, build]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Get short sha
+        id: short_sha
+        run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
       - name: Generate a token
         id: generate-token
         uses: actions/create-github-app-token@v1
@@ -85,10 +128,10 @@ jobs:
               repo: 'dust-infra',
               event_type: 'trigger-component-deploy',
               client_payload: {
-                regions: 'us-central1',
+                regions: '${{ github.event.inputs.regions }}',
                 component: 'front-edge',
                 image_tag: '${{ steps.short_sha.outputs.short_sha }}',
-                slack_thread_ts: "${{ steps.build_message.outputs.thread_ts }}",
+                slack_thread_ts: "${{ needs.notify-start.outputs.thread_ts }}",
                 slack_channel: '${{ secrets.SLACK_CHANNEL_ID }}'
               }
             });
@@ -103,4 +146,4 @@ jobs:
           image_tag: ${{ steps.short_sha.outputs.short_sha }}
           channel: ${{ secrets.SLACK_CHANNEL_ID }}
           slack_token: ${{ secrets.SLACK_BOT_TOKEN }}
-          thread_ts: "${{ steps.build_message.outputs.thread_ts }}"
+          thread_ts: "${{ needs.notify-start.outputs.thread_ts }}"
diff --git a/.github/workflows/deploy-front-qa.yml b/.github/workflows/deploy-front-qa.yml
@@ -45,7 +45,8 @@ jobs:
             --image-name=front-qa \
             --dockerfile-path=./front/Dockerfile \
             --working-dir=./ \
-            --dust-client-facing-url=https://front-qa.dust.tt
+            --dust-client-facing-url=https://front-qa.dust.tt \ 
+            --region=us-central1
 
       - name: Generate a token
         id: generate-token

diff --git a/.github/workflows/deploy-front.yml b/.github/workflows/deploy-front.yml
@@ -64,7 +64,8 @@ jobs:
             --image-name=front \
             --dockerfile-path=./front/Dockerfile \
             --working-dir=./ \
-            --dust-client-facing-url=https://dust.tt
+            --dust-client-facing-url=https://dust.tt \ 
+            --region=us-central1
 
       - name: Generate a token
         id: generate-token

diff --git a/.github/workflows/deploy-nginx.yml b/.github/workflows/deploy-nginx.yml
@@ -51,7 +51,8 @@ jobs:
             --dockerfile-path=./nginx.Dockerfile \
             --working-dir=./ \
             --gcloud-ignore-file=../.gcloudignore-nginx \
-            --dust-client-facing-url=https://dust.tt
+            --dust-client-facing-url=https://dust.tt \ 
+            --region=us-central1
           cd ..
 
       - name: Generate a token

diff --git a/.github/workflows/deploy-oauth.yml b/.github/workflows/deploy-oauth.yml
@@ -64,7 +64,8 @@ jobs:
             --image-name=oauth \
             --dockerfile-path=./oauth.Dockerfile \
             --working-dir=./core/ \
-            --dust-client-facing-url=https://dust.tt
+            --dust-client-facing-url=https://dust.tt \ 
+            --region=us-central1
 
       - name: Generate a token
         id: generate-token

diff --git a/.github/workflows/deploy-prodbox.yml b/.github/workflows/deploy-prodbox.yml
@@ -69,7 +69,8 @@ jobs:
             --dockerfile-path=./prodbox.Dockerfile \
             --working-dir=./ \
             --gcloud-ignore-file=.gcloudignore-prodbox \
-            --dust-client-facing-url=https://dust.tt
+            --dust-client-facing-url=https://dust.tt \ 
+            --region=us-central1
 
       - name: Generate a token
         id: generate-token
@@ -109,4 +110,4 @@ jobs:
           image_tag: ${{ steps.short_sha.outputs.short_sha }}
           channel: ${{ secrets.SLACK_CHANNEL_ID }}
           slack_token: ${{ secrets.SLACK_BOT_TOKEN }}
-          thread_ts: "${{ steps.build_message.outputs.thread_ts }}"
+          thread_ts: "${{ steps.build_message.outputs.thread_ts }}"
diff --git a/.github/workflows/deploy-viz.yml b/.github/workflows/deploy-viz.yml
@@ -67,7 +67,8 @@ jobs:
             --image-name=viz \
             --dockerfile-path=./viz/Dockerfile \
             --working-dir=./ \
-            --dust-client-facing-url=https://dust.tt
+            --dust-client-facing-url=https://dust.tt \ 
+            --region=us-central1
 
       - name: Generate a token
         id: generate-token

diff --git a/connectors/src/connectors/microsoft/temporal/activities.ts b/connectors/src/connectors/microsoft/temporal/activities.ts
@@ -925,7 +925,7 @@ export async function microsoftGarbageCollectionActivity({
 
   const nodes = await MicrosoftNodeResource.fetchByPaginatedIds({
     connectorId,
-    pageSize: 300,
+    pageSize: 500,
     idCursor,
   });
 

diff --git a/connectors/src/connectors/microsoft/temporal/workflows.ts b/connectors/src/connectors/microsoft/temporal/workflows.ts
@@ -197,7 +197,7 @@ export async function microsoftGarbageCollectionWorkflow({
       rootNodeIds,
       startGarbageCollectionTs,
     });
-    await sleep("1 minute");
+    await sleep("30 seconds");
   }
 }
 

diff --git a/connectors/src/connectors/slack/temporal/activities.ts b/connectors/src/connectors/slack/temporal/activities.ts
@@ -588,7 +588,7 @@ export async function syncNonThreaded(
     documentUrl: sourceUrl,
     timestampMs: updatedAt,
     tags,
-    parents: [documentId, channelId],
+    parents: [documentId, channelId, `slack-channel-${channelId}`],
     upsertContext: {
       sync_type: isBatchSync ? "batch" : "incremental",
     },
@@ -782,7 +782,7 @@ export async function syncThread(
     documentUrl: sourceUrl,
     timestampMs: updatedAt,
     tags,
-    parents: [documentId, channelId],
+    parents: [documentId, channelId, `slack-channel-${channelId}`],
     upsertContext: {
       sync_type: isBatchSync ? "batch" : "incremental",
     },

diff --git a/connectors/src/connectors/zendesk/lib/cli.ts b/connectors/src/connectors/zendesk/lib/cli.ts
@@ -73,6 +73,7 @@ export const zendesk = async ({
         brandSubdomain,
         accessToken,
         retentionPeriodDays,
+        query: args.query || null,
       });
       logger.info(
         { connectorId, brandId, ticketCount, retentionPeriodDays },

diff --git a/connectors/src/connectors/zendesk/lib/zendesk_api.ts b/connectors/src/connectors/zendesk/lib/zendesk_api.ts
@@ -384,15 +384,18 @@ export async function fetchZendeskTicketCount({
   accessToken,
   brandSubdomain,
   retentionPeriodDays,
+  query = null,
 }: {
   brandSubdomain: string;
   accessToken: string;
-  retentionPeriodDays: number;
-}): Promise<number> {
-  const query = `type:ticket status:solved updated>${retentionPeriodDays}days`;
+} & (
+  | { retentionPeriodDays?: number; query: string }
+  | { retentionPeriodDays: number; query?: null }
+)): Promise<number> {
+  query ||= `type:ticket status:solved updated>${retentionPeriodDays}days`;
   const url = `https://${brandSubdomain}.zendesk.com/api/v2/search/count?query=${encodeURIComponent(query)}`;
   const response = await fetchFromZendeskWithRetries({ url, accessToken });
-  return Number(response.count);
+  return parseInt(response.count, 10);
 }
 
 /**

diff --git a/core/bin/elasticsearch/create_index.rs b/core/bin/elasticsearch/create_index.rs
@@ -50,7 +50,7 @@ struct Args {
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // parse args and env vars
     let args = Args::parse();
-    let index_alias = args.index_name;
+    let index_name = args.index_name;
     let index_version = args.index_version;
 
     let url = std::env::var("ELASTICSEARCH_URL").expect("ELASTICSEARCH_URL must be set");
@@ -66,12 +66,16 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let u = Url::parse(&url)?;
     let conn_pool = SingleNodeConnectionPool::new(u);
     let mut transport_builder = TransportBuilder::new(conn_pool);
-    transport_builder = transport_builder.auth(credentials);
+    transport_builder = transport_builder
+        .auth(credentials)
+        .disable_proxy()
+        .cert_validation(elasticsearch::cert::CertificateValidation::None);
     let transport = transport_builder.build()?;
 
     let client = Elasticsearch::new(transport);
 
-    let index_fullname = format!("core.{}_{}", index_alias, index_version);
+    let index_fullname = format!("core.{}_{}", index_name, index_version);
+    let index_alias = format!("core.{}", index_name);
 
     // do not create index if it already exists
     let response = client
@@ -87,13 +91,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // get index settings and mappings, parse them as json
     let settings_path = format!(
         "src/search_stores/indices/{}_{}.settings.{}.json",
-        index_alias,
+        index_name,
         index_version,
         region.to_string().to_lowercase()
     );
     let mappings_path = format!(
         "src/search_stores/indices/{}_{}.mappings.json",
-        index_alias, index_version
+        index_name, index_version
     );
 
     // catch errors, provide the error message

diff --git a/core/src/lib.rs b/core/src/lib.rs
@@ -51,6 +51,7 @@ pub mod providers {
     }
     pub mod anthropic;
     pub mod google_ai_studio;
+    pub mod togetherai;
 }
 pub mod http {
     pub mod request;