[connectors] Implement articles pagination for Zendesk (#8485)

* fix the url passed in the articles

* refactor: group updatable fields when updating/creating a new article in db

* feat: when setting a subdomain, fetches the brand from the db if found

* feat: implement pagination for the articles within a category

* fix: prevent category data from leaking through workflows

This change prevents user data from being exposed to Temporal, replacing them with IDs at the cost of additional fetches to the db.

* refactor: rename the `token` parameter into `accessToken` in `createZendeskClient` for consistency

* 📖

* refactor: simplify 2 huge logging calls

* refactor: remove a duplicate variable

* fix: remove an obsolete default value

* add a max sleep time of 1 min

* add a logging entry whenever the rate limit is hit

* add a max number of retries against the rate limit

* refactor: rewrite infinite loops into while loops

* fix: fix the name of the field in the response output

* feat: add a throw if the retryAfter is too big, add a min value

* prevent a ZendeskClient from being instantiated if not needed in allowSyncZendeskCategory

* docs: add function description

* fix: retrieve the correct brands when retrieving permissions

We used to look for brands with a Help Center only, instead of fetching all brands with read permissions.

* fix: show brand that do not have a help center

* refactor: update the return type of the brand sync methods to only return a boolean

* refactor: add a method that fetches a brand and syncs it

* refactor: replace fetchBrandAndSync with syncBrandWithPermissions that does the db fetch

* refactor: add methods to grant permissions for consistency over the revoke

* fix: prevent a regression where extra calls to OAuth would be made even when not necessary

* refactor: simplify the case where the brand is in db in `syncBrandWithPermissions`

connectors/src/connectors/zendesk/index.ts

@@ -252,13 +252,13 @@ export class ZendeskConnectorManager extends BaseConnectorManager<null> {
             }
           }
           if (permission === "read") {
-            const newBrand = await allowSyncZendeskHelpCenter({
+            const wasBrandUpdated = await allowSyncZendeskHelpCenter({
               connectorId,
               connectionId,
               brandId: objectId,
             });
-            if (newBrand) {
-              toBeSignaledHelpCenterIds.add(newBrand.brandId);
+            if (wasBrandUpdated) {
+              toBeSignaledHelpCenterIds.add(objectId);
             }
           }
           break;
@@ -274,13 +274,13 @@ export class ZendeskConnectorManager extends BaseConnectorManager<null> {
             }
           }
           if (permission === "read") {
-            const newBrand = await allowSyncZendeskTickets({
+            const wasBrandUpdated = await allowSyncZendeskTickets({
               connectorId,
               connectionId,
               brandId: objectId,
             });
-            if (newBrand) {
-              toBeSignaledTicketsIds.add(newBrand.brandId);
+            if (wasBrandUpdated) {
+              toBeSignaledTicketsIds.add(objectId);
             }
           }
           break;

connectors/src/connectors/zendesk/lib/brand_permissions.ts

@@ -2,8 +2,7 @@ import type { ModelId } from "@dust-tt/types";
 
 import { allowSyncZendeskHelpCenter } from "@connectors/connectors/zendesk/lib/help_center_permissions";
 import { allowSyncZendeskTickets } from "@connectors/connectors/zendesk/lib/ticket_permissions";
-import { getZendeskSubdomainAndAccessToken } from "@connectors/connectors/zendesk/lib/zendesk_access_token";
-import { createZendeskClient } from "@connectors/connectors/zendesk/lib/zendesk_api";
+import { syncBrandWithPermissions } from "@connectors/connectors/zendesk/lib/utils";
 import logger from "@connectors/logger/logger";
 import { ZendeskBrandResource } from "@connectors/resources/zendesk_resources";
 
@@ -18,46 +17,18 @@ export async function allowSyncZendeskBrand({
   connectorId: ModelId;
   connectionId: string;
   brandId: number;
-}): Promise<ZendeskBrandResource | null> {
-  let brand = await ZendeskBrandResource.fetchByBrandId({
+}): Promise<boolean> {
+  const syncSuccess = await syncBrandWithPermissions({
     connectorId,
+    connectionId,
     brandId,
+    permissions: {
+      ticketsPermission: "none",
+      helpCenterPermission: "read",
+    },
   });
-  if (brand?.helpCenterPermission === "none") {
-    await brand.update({ helpCenterPermission: "read" });
-  }
-  if (brand?.ticketsPermission === "none") {
-    await brand.update({ ticketsPermission: "read" });
-  }
-
-  const { accessToken, subdomain } =
-    await getZendeskSubdomainAndAccessToken(connectionId);
-  const zendeskApiClient = createZendeskClient({
-    token: accessToken,
-    subdomain,
-  });
-
-  if (!brand) {
-    const {
-      result: { brand: fetchedBrand },
-    } = await zendeskApiClient.brand.show(brandId);
-    if (fetchedBrand) {
-      brand = await ZendeskBrandResource.makeNew({
-        blob: {
-          subdomain: fetchedBrand.subdomain,
-          connectorId: connectorId,
-          brandId: fetchedBrand.id,
-          name: fetchedBrand.name || "Brand",
-          helpCenterPermission: "read",
-          ticketsPermission: "read",
-          hasHelpCenter: fetchedBrand.has_help_center,
-          url: fetchedBrand.url,
-        },
-      });
-    } else {
-      logger.error({ brandId }, "[Zendesk] Brand could not be fetched.");
-      return null;
-    }
+  if (!syncSuccess) {
+    return false; // stopping early if the brand sync failed
   }
 
   await allowSyncZendeskHelpCenter({
@@ -71,7 +42,7 @@ export async function allowSyncZendeskBrand({
     brandId,
   });
 
-  return brand;
+  return true;
 }
 
 /**

connectors/src/connectors/zendesk/lib/help_center_permissions.ts

@@ -1,5 +1,6 @@
 import type { ModelId } from "@dust-tt/types";
 
+import { syncBrandWithPermissions } from "@connectors/connectors/zendesk/lib/utils";
 import { getZendeskSubdomainAndAccessToken } from "@connectors/connectors/zendesk/lib/zendesk_access_token";
 import {
   changeZendeskClientSubdomain,
@@ -25,52 +26,31 @@ export async function allowSyncZendeskHelpCenter({
   connectionId: string;
   brandId: number;
   withChildren?: boolean;
-}): Promise<ZendeskBrandResource | null> {
-  let brand = await ZendeskBrandResource.fetchByBrandId({
+}): Promise<boolean> {
+  const zendeskApiClient = createZendeskClient(
+    await getZendeskSubdomainAndAccessToken(connectionId)
+  );
+
+  const syncSuccess = await syncBrandWithPermissions({
+    zendeskApiClient,
+    connectionId,
     connectorId,
     brandId,
+    permissions: {
+      ticketsPermission: "none",
+      helpCenterPermission: "read",
+    },
   });
-
-  if (brand?.helpCenterPermission === "none") {
-    await brand.update({ helpCenterPermission: "read" });
-  }
-
-  const { accessToken, subdomain } =
-    await getZendeskSubdomainAndAccessToken(connectionId);
-  const zendeskApiClient = createZendeskClient({
-    token: accessToken,
-    subdomain,
-  });
-
-  if (!brand) {
-    const {
-      result: { brand: fetchedBrand },
-    } = await zendeskApiClient.brand.show(brandId);
-    if (fetchedBrand) {
-      brand = await ZendeskBrandResource.makeNew({
-        blob: {
-          subdomain: fetchedBrand.subdomain,
-          connectorId: connectorId,
-          brandId: fetchedBrand.id,
-          name: fetchedBrand.name || "Brand",
-          helpCenterPermission: "read",
-          ticketsPermission: "none",
-          hasHelpCenter: fetchedBrand.has_help_center,
-          url: fetchedBrand.url,
-        },
-      });
-    } else {
-      logger.error(
-        { connectorId, brandId },
-        "[Zendesk] Brand could not be fetched."
-      );
-      return null;
-    }
+  if (!syncSuccess) {
+    return false; // stopping early if the brand sync failed
   }
 
   // updating permissions for all the children categories
   if (withChildren) {
-    await changeZendeskClientSubdomain({ client: zendeskApiClient, brandId });
+    await changeZendeskClientSubdomain(zendeskApiClient, {
+      connectorId,
+      brandId,
+    });
     try {
       const categories = await zendeskApiClient.helpcenter.categories.list();
       categories.forEach((category) =>
@@ -86,11 +66,11 @@ export async function allowSyncZendeskHelpCenter({
         { connectorId, brandId },
         "[Zendesk] Categories could not be fetched."
       );
-      return null;
+      return false;
     }
   }
 
-  return brand;
+  return true;
 }
 
 /**
@@ -157,15 +137,14 @@ export async function allowSyncZendeskCategory({
     await category.update({ permission: "read" });
   }
 
-  const { accessToken, subdomain } =
-    await getZendeskSubdomainAndAccessToken(connectionId);
-  const zendeskApiClient = createZendeskClient({
-    token: accessToken,
-    subdomain,
-  });
-
   if (!category) {
-    await changeZendeskClientSubdomain({ client: zendeskApiClient, brandId });
+    const zendeskApiClient = createZendeskClient(
+      await getZendeskSubdomainAndAccessToken(connectionId)
+    );
+    await changeZendeskClientSubdomain(zendeskApiClient, {
+      connectorId,
+      brandId,
+    });
     const { result: fetchedCategory } =
       await zendeskApiClient.helpcenter.categories.show(categoryId);
     if (fetchedCategory) {

connectors/src/connectors/zendesk/lib/permissions.ts

@@ -78,38 +78,33 @@ export async function retrieveChildrenNodes({
   const isReadPermissionsOnly = filterPermission === "read";
   let nodes: ContentNode[] = [];
 
-  const { accessToken, subdomain } = await getZendeskSubdomainAndAccessToken(
-    connector.connectionId
+  const zendeskApiClient = createZendeskClient(
+    await getZendeskSubdomainAndAccessToken(connector.connectionId)
   );
-  const zendeskApiClient = createZendeskClient({
-    token: accessToken,
-    subdomain,
-  });
 
   // At the root level, we show one node for each brand.
   if (!parentInternalId) {
     if (isReadPermissionsOnly) {
-      const brandsInDatabase =
-        await ZendeskBrandResource.fetchAllWithHelpCenter({ connectorId });
+      const brandsInDatabase = await ZendeskBrandResource.fetchAllReadOnly({
+        connectorId,
+      });
       nodes = brandsInDatabase.map((brand) =>
         brand.toContentNode({ connectorId })
       );
     } else {
       const { result: brands } = await zendeskApiClient.brand.list();
-      nodes = brands
-        .filter((brand) => brand.has_help_center)
-        .map((brand) => ({
-          provider: connector.type,
-          internalId: getBrandInternalId(connectorId, brand.id),
-          parentInternalId: null,
-          type: "folder",
-          title: brand.name || "Brand",
-          sourceUrl: brand.brand_url,
-          expandable: true,
-          permission: "none",
-          dustDocumentId: null,
-          lastUpdatedAt: null,
-        }));
+      nodes = brands.map((brand) => ({
+        provider: connector.type,
+        internalId: getBrandInternalId(connectorId, brand.id),
+        parentInternalId: null,
+        type: "folder",
+        title: brand.name || "Brand",
+        sourceUrl: brand.brand_url,
+        expandable: true,
+        permission: "none",
+        dustDocumentId: null,
+        lastUpdatedAt: null,
+      }));
     }
   } else {
     const { type, objectId } = getIdFromInternalId(
@@ -188,8 +183,8 @@ export async function retrieveChildrenNodes({
             category.toContentNode({ connectorId, expandable: true })
           );
         } else {
-          await changeZendeskClientSubdomain({
-            client: zendeskApiClient,
+          await changeZendeskClientSubdomain(zendeskApiClient, {
+            connectorId,
             brandId: objectId,
           });
           const categories =

connectors/src/connectors/zendesk/lib/ticket_permissions.ts

@@ -1,13 +1,15 @@
 import type { ModelId } from "@dust-tt/types";
 
-import { getZendeskSubdomainAndAccessToken } from "@connectors/connectors/zendesk/lib/zendesk_access_token";
-import { createZendeskClient } from "@connectors/connectors/zendesk/lib/zendesk_api";
+import { syncBrandWithPermissions } from "@connectors/connectors/zendesk/lib/utils";
 import logger from "@connectors/logger/logger";
 import {
   ZendeskBrandResource,
   ZendeskTicketResource,
 } from "@connectors/resources/zendesk_resources";
 
+/**
+ * Marks the node "Tickets" of a Brand as permission "read".
+ */
 export async function allowSyncZendeskTickets({
   connectorId,
   connectionId,
@@ -16,50 +18,20 @@ export async function allowSyncZendeskTickets({
   connectorId: ModelId;
   connectionId: string;
   brandId: number;
-}): Promise<ZendeskBrandResource | null> {
-  let brand = await ZendeskBrandResource.fetchByBrandId({
+}): Promise<boolean> {
+  return syncBrandWithPermissions({
     connectorId,
+    connectionId,
     brandId,
+    permissions: {
+      ticketsPermission: "read",
+      helpCenterPermission: "none",
+    },
   });
-  if (brand?.ticketsPermission === "none") {
-    await brand.update({ ticketsPermission: "read" });
-  }
-
-  const { accessToken, subdomain } =
-    await getZendeskSubdomainAndAccessToken(connectionId);
-  const zendeskApiClient = createZendeskClient({
-    token: accessToken,
-    subdomain,
-  });
-
-  if (!brand) {
-    const {
-      result: { brand: fetchedBrand },
-    } = await zendeskApiClient.brand.show(brandId);
-    if (fetchedBrand) {
-      brand = await ZendeskBrandResource.makeNew({
-        blob: {
-          subdomain: fetchedBrand.subdomain,
-          connectorId: connectorId,
-          brandId: fetchedBrand.id,
-          name: fetchedBrand.name || "Brand",
-          helpCenterPermission: "none",
-          ticketsPermission: "read",
-          hasHelpCenter: fetchedBrand.has_help_center,
-          url: fetchedBrand.url,
-        },
-      });
-    } else {
-      logger.error({ brandId }, "[Zendesk] Brand could not be fetched.");
-      return null;
-    }
-  }
-
-  return brand;
 }
 
 /**
- * Mark a help center as permission "none" and all children (collections and articles).
+ * Mark the node "Tickets" and all the children tickets for a Brand as permission "none".
  */
 export async function revokeSyncZendeskTickets({
   connectorId,