diff --git a/connectors/src/connectors/zendesk/lib/node-zendesk-types.d.ts b/connectors/src/connectors/zendesk/lib/node-zendesk-types.d.ts index 09a91f0b387f..20c6fab57d41 100644 --- a/connectors/src/connectors/zendesk/lib/node-zendesk-types.d.ts +++ b/connectors/src/connectors/zendesk/lib/node-zendesk-types.d.ts @@ -2,7 +2,7 @@ import "node-zendesk"; import type { ZendeskClientOptions } from "node-zendesk"; -interface Brand { +interface ZendeskFetchedBrand { url: string; id: number; name: string; @@ -27,7 +27,7 @@ interface Response { statusText: string; } -interface Category { +interface ZendeskFetchedCategory { id: number; url: string; html_url: string; @@ -41,7 +41,7 @@ interface Category { outdated: boolean; } -interface Article { +export interface ZendeskFetchedArticle { id: number; url: string; html_url: string; @@ -70,7 +70,7 @@ interface Article { user_segment_ids: number[]; } -interface Ticket { +interface ZendeskFetchedTicket { assignee_id: number; collaborator_ids: number[]; created_at: string; // ISO 8601 date string @@ -93,6 +93,7 @@ interface Ticket { problem_id: number; raw_subject: string; recipient: string; + requester: { locale_id: number; name: string; email: string }; requester_id: number; satisfaction_rating: { comment: string; @@ -100,11 +101,11 @@ interface Ticket { score: string; }; sharing_agreement_ids: number[]; - status: string; + status: "new" | "open" | "pending" | "hold" | "solved" | "closed"; subject: string; submitter_id: number; tags: string[]; - type: string; + type: "problem" | "incident" | "question" | "task"; updated_at: string; // ISO 8601 date string url: string; via: { @@ -118,34 +119,39 @@ declare module "node-zendesk" { brand: { list: () => Promise<{ response: Response; - result: Brand[]; + result: ZendeskFetchedBrand[]; + }>; + show: (brandId: number) => Promise<{ + response: Response; + result: { brand: ZendeskFetchedBrand }; }>; - show: ( - brandId: number - ) => Promise<{ response: Response; result: { brand: Brand } }>; }; helpcenter: { categories: { - list: () => Promise; + list: () => Promise; show: ( categoryId: number - ) => Promise<{ response: Response; result: Category }>; + ) => Promise<{ response: Response; result: ZendeskFetchedCategory }>; }; articles: { - list: () => Promise; + list: () => Promise; show: ( articleId: number - ) => Promise<{ response: Response; result: Article }>; - listByCategory: (categoryId: number) => Promise; - listSinceStartTime: (startTime: number) => Promise; - }; - tickets: { - list: () => Promise; - show: ( - ticketId: number - ) => Promise<{ response: Response; result: Ticket }>; + ) => Promise<{ response: Response; result: ZendeskFetchedArticle }>; + listByCategory: ( + categoryId: number + ) => Promise; + listSinceStartTime: ( + startTime: number + ) => Promise; }; }; + tickets: { + list: () => Promise; + show: ( + ticketId: number + ) => Promise<{ response: Response; result: ZendeskFetchedTicket }>; + }; } export function createClient(options: object): Client; diff --git a/connectors/src/connectors/zendesk/temporal/activities.ts b/connectors/src/connectors/zendesk/temporal/activities.ts index d9236fdfb96b..bf9b2f1c9df3 100644 --- a/connectors/src/connectors/zendesk/temporal/activities.ts +++ b/connectors/src/connectors/zendesk/temporal/activities.ts @@ -9,7 +9,10 @@ import { changeZendeskClientSubdomain, createZendeskClient, } from "@connectors/connectors/zendesk/lib/zendesk_api"; +import { syncArticle } from "@connectors/connectors/zendesk/temporal/sync_article"; +import { syncTicket } from "@connectors/connectors/zendesk/temporal/sync_ticket"; import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config"; +import { concurrentExecutor } from "@connectors/lib/async_utils"; import { deleteFromDataSource } from "@connectors/lib/data_sources"; import { syncStarted, syncSucceeded } from "@connectors/lib/sync_status"; import { ConnectorResource } from "@connectors/resources/connector_resource"; @@ -39,6 +42,25 @@ async function _getZendeskConfigurationOrRaise(connectorId: ModelId) { return configuration; } +async function _getZendeskCategoryOrRaise({ + connectorId, + categoryId, +}: { + connectorId: ModelId; + categoryId: number; +}) { + const category = await ZendeskCategoryResource.fetchByCategoryId({ + connectorId, + categoryId, + }); + if (!category) { + throw new Error( + `[Zendesk] Category not found, connectorId: ${connectorId}, categoryId: ${categoryId}` + ); + } + return category; +} + /** * This activity is responsible for updating the lastSyncStartTime of the connector to now. */ @@ -249,15 +271,10 @@ export async function syncZendeskCategoryActivity({ const connector = await _getZendeskConnectorOrRaise(connectorId); const dataSourceConfig = dataSourceConfigFromConnector(connector); const configuration = await _getZendeskConfigurationOrRaise(connectorId); - const categoryInDb = await ZendeskCategoryResource.fetchByCategoryId({ + const categoryInDb = await _getZendeskCategoryOrRaise({ connectorId, categoryId, }); - if (!categoryInDb) { - throw new Error( - `[Zendesk] Category not found, connectorId: ${connectorId}, categoryId: ${categoryId}` - ); - } // if all rights were revoked, we delete the category data. if (categoryInDb.permission === "none") { @@ -292,29 +309,105 @@ export async function syncZendeskCategoryActivity({ /** * This activity is responsible for syncing all the articles in a Category. * It does not sync the Category, only the Articles. - * - * @returns true if the Category was updated, false if it was deleted. */ -// eslint-disable-next-line no-empty-pattern -export async function syncZendeskArticlesActivity({}: { +export async function syncZendeskArticlesActivity({ + connectorId, + categoryId, + currentSyncDateMs, + forceResync, +}: { connectorId: ModelId; categoryId: number; currentSyncDateMs: number; forceResync: boolean; -}) {} +}) { + const connector = await _getZendeskConnectorOrRaise(connectorId); + const dataSourceConfig = dataSourceConfigFromConnector(connector); + const loggerArgs = { + workspaceId: dataSourceConfig.workspaceId, + connectorId, + provider: "zendesk", + dataSourceId: dataSourceConfig.dataSourceId, + }; + + const [configuration, categoryInDb, accessToken] = await Promise.all([ + _getZendeskConfigurationOrRaise(connectorId), + _getZendeskCategoryOrRaise({ connectorId, categoryId }), + getZendeskAccessToken(connector.connectionId), + ]); + const zendeskApiClient = createZendeskClient({ + token: accessToken, + subdomain: configuration.subdomain, + }); + + const articles = + await zendeskApiClient.helpcenter.articles.listByCategory(categoryId); + + await concurrentExecutor( + articles, + (article) => + syncArticle({ + connectorId, + brandId: categoryInDb.brandId, + categoryId, + article, + dataSourceConfig, + currentSyncDateMs, + loggerArgs, + forceResync, + }), + { concurrency: 10 } + ); +} /** * This activity is responsible for syncing all the tickets for a Brand. */ -// eslint-disable-next-line no-empty-pattern -export async function syncZendeskTicketsActivity({}: { +export async function syncZendeskTicketsActivity({ + connectorId, + brandId, + currentSyncDateMs, + forceResync, +}: { connectorId: ModelId; brandId: number; currentSyncDateMs: number; forceResync: boolean; - afterCursor: string | null; -}): Promise<{ hasMore: boolean; afterCursor: string }> { - return { hasMore: false, afterCursor: "" }; +}) { + const connector = await _getZendeskConnectorOrRaise(connectorId); + const dataSourceConfig = dataSourceConfigFromConnector(connector); + const loggerArgs = { + workspaceId: dataSourceConfig.workspaceId, + connectorId, + provider: "zendesk", + dataSourceId: dataSourceConfig.dataSourceId, + }; + const [configuration, accessToken] = await Promise.all([ + _getZendeskConfigurationOrRaise(connectorId), + getZendeskAccessToken(connector.connectionId), + ]); + + const zendeskApiClient = createZendeskClient({ + token: accessToken, + subdomain: configuration.subdomain, + }); + await changeZendeskClientSubdomain({ client: zendeskApiClient, brandId }); + const tickets = await zendeskApiClient.tickets.list(); + + await concurrentExecutor( + tickets, + (ticket) => + syncTicket({ + connectorId, + brandId, + ticket, + dataSourceConfig, + currentSyncDateMs, + loggerArgs, + forceResync, + }), + { concurrency: 10 } + ); } /** diff --git a/connectors/src/connectors/zendesk/temporal/sync_article.ts b/connectors/src/connectors/zendesk/temporal/sync_article.ts new file mode 100644 index 000000000000..ade8ab9441cd --- /dev/null +++ b/connectors/src/connectors/zendesk/temporal/sync_article.ts @@ -0,0 +1,56 @@ +import type { ModelId } from "@dust-tt/types"; + +import type { ZendeskFetchedArticle } from "@connectors/connectors/zendesk/lib/node-zendesk-types"; +import { ZendeskArticleResource } from "@connectors/resources/zendesk_resources"; +import type { DataSourceConfig } from "@connectors/types/data_source_config"; + +export async function syncArticle({ + connectorId, + article, + brandId, + categoryId, + currentSyncDateMs, +}: { + connectorId: ModelId; + dataSourceConfig: DataSourceConfig; + article: ZendeskFetchedArticle; + brandId: number; + categoryId: number; + currentSyncDateMs: number; + loggerArgs: Record; + forceResync: boolean; +}) { + let articleInDb = await ZendeskArticleResource.fetchByArticleId({ + connectorId, + articleId: article.id, + }); + const createdAtDate = new Date(article.created_at); + const updatedAtDate = new Date(article.updated_at); + + if (!articleInDb) { + articleInDb = await ZendeskArticleResource.makeNew({ + blob: { + createdAt: createdAtDate, + updatedAt: updatedAtDate, + articleId: article.id, + brandId, + categoryId, + permission: "read", + name: article.name, + url: article.url, + lastUpsertedTs: new Date(currentSyncDateMs), + connectorId, + }, + }); + } else { + await articleInDb.update({ + createdAt: createdAtDate, + updatedAt: updatedAtDate, + categoryId, // an article can be moved from one category to another, which does not apply to brands + name: article.name, + url: article.url, + lastUpsertedTs: new Date(currentSyncDateMs), + }); + } + /// TODO: upsert the article here +} diff --git a/connectors/src/connectors/zendesk/temporal/sync_ticket.ts b/connectors/src/connectors/zendesk/temporal/sync_ticket.ts new file mode 100644 index 000000000000..fa968a269adb --- /dev/null +++ b/connectors/src/connectors/zendesk/temporal/sync_ticket.ts @@ -0,0 +1,75 @@ +import type { ModelId } from "@dust-tt/types"; + +import type { ZendeskFetchedTicket } from "@connectors/connectors/zendesk/lib/node-zendesk-types"; +import { ZendeskTicketResource } from "@connectors/resources/zendesk_resources"; +import type { DataSourceConfig } from "@connectors/types/data_source_config"; + +export async function syncTicket({ + connectorId, + ticket, + brandId, + currentSyncDateMs, +}: { + connectorId: ModelId; + dataSourceConfig: DataSourceConfig; + ticket: ZendeskFetchedTicket; + brandId: number; + currentSyncDateMs: number; + loggerArgs: Record; + forceResync: boolean; +}) { + let ticketInDb = await ZendeskTicketResource.fetchByTicketId({ + connectorId, + ticketId: ticket.id, + }); + const createdAtDate = new Date(ticket.created_at); + const updatedAtDate = new Date(ticket.updated_at); + + if (!ticketInDb) { + ticketInDb = await ZendeskTicketResource.makeNew({ + blob: { + createdAt: createdAtDate, + updatedAt: updatedAtDate, + ticketId: ticket.id, + brandId, + permission: "read", + assigneeId: ticket.assignee_id, + groupId: ticket.group_id, + organizationId: ticket.organization_id, + name: "Ticket", + description: ticket.description, + subject: ticket.subject, + requesterMail: ticket?.requester?.email || "", + url: ticket.url, + lastUpsertedTs: new Date(currentSyncDateMs), + satisfactionScore: ticket.satisfaction_rating?.score || "unknown", + satisfactionComment: ticket.satisfaction_rating?.comment || "", + status: ticket.status, + tags: ticket.tags, + type: ticket.type, + customFields: ticket.custom_fields.map((field) => field.value), + connectorId, + }, + }); + } else { + await ticketInDb.update({ + createdAt: createdAtDate, + updatedAt: updatedAtDate, + assigneeId: ticket.assignee_id, + groupId: ticket.group_id, + organizationId: ticket.organization_id, + description: ticket.description, + subject: ticket.subject, + requesterMail: ticket?.requester?.email || "", + url: ticket.url, + satisfactionScore: ticket.satisfaction_rating?.score || "unknown", + satisfactionComment: ticket.satisfaction_rating?.comment || "", + status: ticket.status, + tags: ticket.tags, + type: ticket.type, + customFields: ticket.custom_fields.map((field) => field.value), + lastUpsertedTs: new Date(currentSyncDateMs), + }); + } + /// TODO: upsert the ticket here +} diff --git a/connectors/src/connectors/zendesk/temporal/workflows.ts b/connectors/src/connectors/zendesk/temporal/workflows.ts index 19fa1cf0b429..af619ecaaed0 100644 --- a/connectors/src/connectors/zendesk/temporal/workflows.ts +++ b/connectors/src/connectors/zendesk/temporal/workflows.ts @@ -360,7 +360,6 @@ async function runZendeskBrandHelpCenterSyncActivities({ /** * Run the activities necessary to sync the Tickets of a Brand. */ -// eslint-disable-next-line no-empty-pattern async function runZendeskBrandTicketsSyncActivities({ connectorId, brandId, @@ -372,15 +371,11 @@ async function runZendeskBrandTicketsSyncActivities({ currentSyncDateMs: number; forceResync: boolean; }) { - let hasMore = true; - let afterCursor = null; - do { - ({ hasMore, afterCursor } = await syncZendeskTicketsActivity({ - connectorId, - brandId, - currentSyncDateMs, - forceResync, - afterCursor, - })); - } while (hasMore); + // TODO(2024-10-29 aubin): see how we can batch the tickets into multiple activities + await syncZendeskTicketsActivity({ + connectorId, + brandId, + currentSyncDateMs, + forceResync, + }); } diff --git a/connectors/src/resources/zendesk_resources.ts b/connectors/src/resources/zendesk_resources.ts index 28eaf09c9e9e..13754aadd934 100644 --- a/connectors/src/resources/zendesk_resources.ts +++ b/connectors/src/resources/zendesk_resources.ts @@ -451,6 +451,20 @@ export class ZendeskTicketResource extends BaseResource { super(ZendeskTicket, blob); } + static async makeNew({ + blob, + transaction, + }: { + blob: CreationAttributes; + transaction?: Transaction; + }): Promise { + const article = await ZendeskTicket.create( + { ...blob }, + transaction && { transaction } + ); + return new this(this.model, article.get()); + } + async postFetchHook(): Promise { return; } @@ -577,6 +591,20 @@ export class ZendeskArticleResource extends BaseResource { super(ZendeskArticle, blob); } + static async makeNew({ + blob, + transaction, + }: { + blob: CreationAttributes; + transaction?: Transaction; + }): Promise { + const article = await ZendeskArticle.create( + { ...blob }, + transaction && { transaction } + ); + return new this(this.model, article.get()); + } + async postFetchHook(): Promise { return; }