[connectors] Add sync activities for Zendesk Articles and Tickets (#8305

)

* refactor: rename the interfaces used to type the responses of the API

* feat: add the db interaction in syncZendeskArticlesActivity

* feat: implement ticket sync (db interaction only, no upsert)

* feat: plug ticket sync on the existing workflow

* chore: remove TODOs to stop Danger from panicking

* perf: parallelize certain awaited promises

* fix: remove an unused configuration

* fix: remove an unused configuration

* refactor: cleanup the makeNew methods

connectors/src/connectors/zendesk/lib/node-zendesk-types.d.ts

@@ -2,7 +2,7 @@ import "node-zendesk";
 
 import type { ZendeskClientOptions } from "node-zendesk";
 
-interface Brand {
+interface ZendeskFetchedBrand {
   url: string;
   id: number;
   name: string;
@@ -27,7 +27,7 @@ interface Response {
   statusText: string;
 }
 
-interface Category {
+interface ZendeskFetchedCategory {
   id: number;
   url: string;
   html_url: string;
@@ -41,7 +41,7 @@ interface Category {
   outdated: boolean;
 }
 
-interface Article {
+export interface ZendeskFetchedArticle {
   id: number;
   url: string;
   html_url: string;
@@ -70,7 +70,7 @@ interface Article {
   user_segment_ids: number[];
 }
 
-interface Ticket {
+interface ZendeskFetchedTicket {
   assignee_id: number;
   collaborator_ids: number[];
   created_at: string; // ISO 8601 date string
@@ -93,18 +93,19 @@ interface Ticket {
   problem_id: number;
   raw_subject: string;
   recipient: string;
+  requester: { locale_id: number; name: string; email: string };
   requester_id: number;
   satisfaction_rating: {
     comment: string;
     id: number;
     score: string;
   };
   sharing_agreement_ids: number[];
-  status: string;
+  status: "new" | "open" | "pending" | "hold" | "solved" | "closed";
   subject: string;
   submitter_id: number;
   tags: string[];
-  type: string;
+  type: "problem" | "incident" | "question" | "task";
   updated_at: string; // ISO 8601 date string
   url: string;
   via: {
@@ -118,34 +119,39 @@ declare module "node-zendesk" {
     brand: {
       list: () => Promise<{
         response: Response;
-        result: Brand[];
+        result: ZendeskFetchedBrand[];
+      }>;
+      show: (brandId: number) => Promise<{
+        response: Response;
+        result: { brand: ZendeskFetchedBrand };
       }>;
-      show: (
-        brandId: number
-      ) => Promise<{ response: Response; result: { brand: Brand } }>;
     };
     helpcenter: {
       categories: {
-        list: () => Promise<Category[]>;
+        list: () => Promise<ZendeskFetchedCategory[]>;
         show: (
           categoryId: number
-        ) => Promise<{ response: Response; result: Category }>;
+        ) => Promise<{ response: Response; result: ZendeskFetchedCategory }>;
       };
       articles: {
-        list: () => Promise<Article[]>;
+        list: () => Promise<ZendeskFetchedArticle[]>;
         show: (
           articleId: number
-        ) => Promise<{ response: Response; result: Article }>;
-        listByCategory: (categoryId: number) => Promise<Article[]>;
-        listSinceStartTime: (startTime: number) => Promise<Article[]>;
-      };
-      tickets: {
-        list: () => Promise<Ticket[]>;
-        show: (
-          ticketId: number
-        ) => Promise<{ response: Response; result: Ticket }>;
+        ) => Promise<{ response: Response; result: ZendeskFetchedArticle }>;
+        listByCategory: (
+          categoryId: number
+        ) => Promise<ZendeskFetchedArticle[]>;
+        listSinceStartTime: (
+          startTime: number
+        ) => Promise<ZendeskFetchedArticle[]>;
       };
     };
+    tickets: {
+      list: () => Promise<ZendeskFetchedTicket[]>;
+      show: (
+        ticketId: number
+      ) => Promise<{ response: Response; result: ZendeskFetchedTicket }>;
+    };
   }
 
   export function createClient(options: object): Client;

connectors/src/connectors/zendesk/temporal/activities.ts

@@ -9,7 +9,10 @@ import {
   changeZendeskClientSubdomain,
   createZendeskClient,
 } from "@connectors/connectors/zendesk/lib/zendesk_api";
+import { syncArticle } from "@connectors/connectors/zendesk/temporal/sync_article";
+import { syncTicket } from "@connectors/connectors/zendesk/temporal/sync_ticket";
 import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config";
+import { concurrentExecutor } from "@connectors/lib/async_utils";
 import { deleteFromDataSource } from "@connectors/lib/data_sources";
 import { syncStarted, syncSucceeded } from "@connectors/lib/sync_status";
 import { ConnectorResource } from "@connectors/resources/connector_resource";
@@ -39,6 +42,25 @@ async function _getZendeskConfigurationOrRaise(connectorId: ModelId) {
   return configuration;
 }
 
+async function _getZendeskCategoryOrRaise({
+  connectorId,
+  categoryId,
+}: {
+  connectorId: ModelId;
+  categoryId: number;
+}) {
+  const category = await ZendeskCategoryResource.fetchByCategoryId({
+    connectorId,
+    categoryId,
+  });
+  if (!category) {
+    throw new Error(
+      `[Zendesk] Category not found, connectorId: ${connectorId}, categoryId: ${categoryId}`
+    );
+  }
+  return category;
+}
+
 /**
  * This activity is responsible for updating the lastSyncStartTime of the connector to now.
  */
@@ -249,15 +271,10 @@ export async function syncZendeskCategoryActivity({
   const connector = await _getZendeskConnectorOrRaise(connectorId);
   const dataSourceConfig = dataSourceConfigFromConnector(connector);
   const configuration = await _getZendeskConfigurationOrRaise(connectorId);
-  const categoryInDb = await ZendeskCategoryResource.fetchByCategoryId({
+  const categoryInDb = await _getZendeskCategoryOrRaise({
     connectorId,
     categoryId,
   });
-  if (!categoryInDb) {
-    throw new Error(
-      `[Zendesk] Category not found, connectorId: ${connectorId}, categoryId: ${categoryId}`
-    );
-  }
 
   // if all rights were revoked, we delete the category data.
   if (categoryInDb.permission === "none") {
@@ -292,29 +309,105 @@ export async function syncZendeskCategoryActivity({
 /**
  * This activity is responsible for syncing all the articles in a Category.
  * It does not sync the Category, only the Articles.
- *
- * @returns true if the Category was updated, false if it was deleted.
  */
-// eslint-disable-next-line no-empty-pattern
-export async function syncZendeskArticlesActivity({}: {
+export async function syncZendeskArticlesActivity({
+  connectorId,
+  categoryId,
+  currentSyncDateMs,
+  forceResync,
+}: {
   connectorId: ModelId;
   categoryId: number;
   currentSyncDateMs: number;
   forceResync: boolean;
-}) {}
+}) {
+  const connector = await _getZendeskConnectorOrRaise(connectorId);
+  const dataSourceConfig = dataSourceConfigFromConnector(connector);
+  const loggerArgs = {
+    workspaceId: dataSourceConfig.workspaceId,
+    connectorId,
+    provider: "zendesk",
+    dataSourceId: dataSourceConfig.dataSourceId,
+  };
+
+  const [configuration, categoryInDb, accessToken] = await Promise.all([
+    _getZendeskConfigurationOrRaise(connectorId),
+    _getZendeskCategoryOrRaise({ connectorId, categoryId }),
+    getZendeskAccessToken(connector.connectionId),
+  ]);
+  const zendeskApiClient = createZendeskClient({
+    token: accessToken,
+    subdomain: configuration.subdomain,
+  });
+
+  const articles =
+    await zendeskApiClient.helpcenter.articles.listByCategory(categoryId);
+
+  await concurrentExecutor(
+    articles,
+    (article) =>
+      syncArticle({
+        connectorId,
+        brandId: categoryInDb.brandId,
+        categoryId,
+        article,
+        dataSourceConfig,
+        currentSyncDateMs,
+        loggerArgs,
+        forceResync,
+      }),
+    { concurrency: 10 }
+  );
+}
 
 /**
  * This activity is responsible for syncing all the tickets for a Brand.
  */
-// eslint-disable-next-line no-empty-pattern
-export async function syncZendeskTicketsActivity({}: {
+export async function syncZendeskTicketsActivity({
+  connectorId,
+  brandId,
+  currentSyncDateMs,
+  forceResync,
+}: {
   connectorId: ModelId;
   brandId: number;
   currentSyncDateMs: number;
   forceResync: boolean;
-  afterCursor: string | null;
-}): Promise<{ hasMore: boolean; afterCursor: string }> {
-  return { hasMore: false, afterCursor: "" };
+}) {
+  const connector = await _getZendeskConnectorOrRaise(connectorId);
+  const dataSourceConfig = dataSourceConfigFromConnector(connector);
+  const loggerArgs = {
+    workspaceId: dataSourceConfig.workspaceId,
+    connectorId,
+    provider: "zendesk",
+    dataSourceId: dataSourceConfig.dataSourceId,
+  };
+  const [configuration, accessToken] = await Promise.all([
+    _getZendeskConfigurationOrRaise(connectorId),
+    getZendeskAccessToken(connector.connectionId),
+  ]);
+
+  const zendeskApiClient = createZendeskClient({
+    token: accessToken,
+    subdomain: configuration.subdomain,
+  });
+  await changeZendeskClientSubdomain({ client: zendeskApiClient, brandId });
+  const tickets = await zendeskApiClient.tickets.list();
+
+  await concurrentExecutor(
+    tickets,
+    (ticket) =>
+      syncTicket({
+        connectorId,
+        brandId,
+        ticket,
+        dataSourceConfig,
+        currentSyncDateMs,
+        loggerArgs,
+        forceResync,
+      }),
+    { concurrency: 10 }
+  );
 }
 
 /**

connectors/src/connectors/zendesk/temporal/sync_article.ts

@@ -0,0 +1,56 @@
+import type { ModelId } from "@dust-tt/types";
+
+import type { ZendeskFetchedArticle } from "@connectors/connectors/zendesk/lib/node-zendesk-types";
+import { ZendeskArticleResource } from "@connectors/resources/zendesk_resources";
+import type { DataSourceConfig } from "@connectors/types/data_source_config";
+
+export async function syncArticle({
+  connectorId,
+  article,
+  brandId,
+  categoryId,
+  currentSyncDateMs,
+}: {
+  connectorId: ModelId;
+  dataSourceConfig: DataSourceConfig;
+  article: ZendeskFetchedArticle;
+  brandId: number;
+  categoryId: number;
+  currentSyncDateMs: number;
+  loggerArgs: Record<string, string | number | null>;
+  forceResync: boolean;
+}) {
+  let articleInDb = await ZendeskArticleResource.fetchByArticleId({
+    connectorId,
+    articleId: article.id,
+  });
+  const createdAtDate = new Date(article.created_at);
+  const updatedAtDate = new Date(article.updated_at);
+
+  if (!articleInDb) {
+    articleInDb = await ZendeskArticleResource.makeNew({
+      blob: {
+        createdAt: createdAtDate,
+        updatedAt: updatedAtDate,
+        articleId: article.id,
+        brandId,
+        categoryId,
+        permission: "read",
+        name: article.name,
+        url: article.url,
+        lastUpsertedTs: new Date(currentSyncDateMs),
+        connectorId,
+      },
+    });
+  } else {
+    await articleInDb.update({
+      createdAt: createdAtDate,
+      updatedAt: updatedAtDate,
+      categoryId, // an article can be moved from one category to another, which does not apply to brands
+      name: article.name,
+      url: article.url,
+      lastUpsertedTs: new Date(currentSyncDateMs),
+    });
+  }
+  /// TODO: upsert the article here
+}