From 3556bdb3ef45412667be6475177dc3890dbc7f93 Mon Sep 17 00:00:00 2001 From: Aubin <60398825+aubin-tchoi@users.noreply.github.com> Date: Fri, 8 Nov 2024 14:49:40 +0100 Subject: [PATCH] [connectors] Implement articles pagination for Zendesk (#8485) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix the url passed in the articles * refactor: group updatable fields when updating/creating a new article in db * feat: when setting a subdomain, fetches the brand from the db if found * feat: implement pagination for the articles within a category * fix: prevent category data from leaking through workflows This change prevents user data from being exposed to Temporal, replacing them with IDs at the cost of additional fetches to the db. * refactor: rename the `token` parameter into `accessToken` in `createZendeskClient` for consistency * 📖 * refactor: simplify 2 huge logging calls * refactor: remove a duplicate variable * fix: remove an obsolete default value * add a max sleep time of 1 min * add a logging entry whenever the rate limit is hit * add a max number of retries against the rate limit * refactor: rewrite infinite loops into while loops * fix: fix the name of the field in the response output * feat: add a throw if the retryAfter is too big, add a min value * prevent a ZendeskClient from being instantiated if not needed in allowSyncZendeskCategory * docs: add function description * fix: retrieve the correct brands when retrieving permissions We used to look for brands with a Help Center only, instead of fetching all brands with read permissions. * fix: show brand that do not have a help center * refactor: update the return type of the brand sync methods to only return a boolean * refactor: add a method that fetches a brand and syncs it * refactor: replace fetchBrandAndSync with syncBrandWithPermissions that does the db fetch * refactor: add methods to grant permissions for consistency over the revoke * fix: prevent a regression where extra calls to OAuth would be made even when not necessary * refactor: simplify the case where the brand is in db in `syncBrandWithPermissions` --- connectors/src/connectors/zendesk/index.ts | 12 +- .../zendesk/lib/brand_permissions.ts | 51 ++---- .../zendesk/lib/help_center_permissions.ts | 77 +++----- .../src/connectors/zendesk/lib/permissions.ts | 43 ++--- .../zendesk/lib/ticket_permissions.ts | 52 ++---- .../src/connectors/zendesk/lib/utils.ts | 73 ++++++++ .../src/connectors/zendesk/lib/zendesk_api.ts | 164 ++++++++++++++++-- .../connectors/zendesk/temporal/activities.ts | 89 +++++----- .../src/connectors/zendesk/temporal/config.ts | 2 +- .../zendesk/temporal/sync_article.ts | 64 +++---- .../connectors/zendesk/temporal/workflows.ts | 59 ++++--- connectors/src/resources/zendesk_resources.ts | 12 ++ 12 files changed, 426 insertions(+), 272 deletions(-) create mode 100644 connectors/src/connectors/zendesk/lib/utils.ts diff --git a/connectors/src/connectors/zendesk/index.ts b/connectors/src/connectors/zendesk/index.ts index a70030f864cf..85de2dc794cb 100644 --- a/connectors/src/connectors/zendesk/index.ts +++ b/connectors/src/connectors/zendesk/index.ts @@ -252,13 +252,13 @@ export class ZendeskConnectorManager extends BaseConnectorManager { } } if (permission === "read") { - const newBrand = await allowSyncZendeskHelpCenter({ + const wasBrandUpdated = await allowSyncZendeskHelpCenter({ connectorId, connectionId, brandId: objectId, }); - if (newBrand) { - toBeSignaledHelpCenterIds.add(newBrand.brandId); + if (wasBrandUpdated) { + toBeSignaledHelpCenterIds.add(objectId); } } break; @@ -274,13 +274,13 @@ export class ZendeskConnectorManager extends BaseConnectorManager { } } if (permission === "read") { - const newBrand = await allowSyncZendeskTickets({ + const wasBrandUpdated = await allowSyncZendeskTickets({ connectorId, connectionId, brandId: objectId, }); - if (newBrand) { - toBeSignaledTicketsIds.add(newBrand.brandId); + if (wasBrandUpdated) { + toBeSignaledTicketsIds.add(objectId); } } break; diff --git a/connectors/src/connectors/zendesk/lib/brand_permissions.ts b/connectors/src/connectors/zendesk/lib/brand_permissions.ts index df011173b8cc..bb806674487b 100644 --- a/connectors/src/connectors/zendesk/lib/brand_permissions.ts +++ b/connectors/src/connectors/zendesk/lib/brand_permissions.ts @@ -2,8 +2,7 @@ import type { ModelId } from "@dust-tt/types"; import { allowSyncZendeskHelpCenter } from "@connectors/connectors/zendesk/lib/help_center_permissions"; import { allowSyncZendeskTickets } from "@connectors/connectors/zendesk/lib/ticket_permissions"; -import { getZendeskSubdomainAndAccessToken } from "@connectors/connectors/zendesk/lib/zendesk_access_token"; -import { createZendeskClient } from "@connectors/connectors/zendesk/lib/zendesk_api"; +import { syncBrandWithPermissions } from "@connectors/connectors/zendesk/lib/utils"; import logger from "@connectors/logger/logger"; import { ZendeskBrandResource } from "@connectors/resources/zendesk_resources"; @@ -18,46 +17,18 @@ export async function allowSyncZendeskBrand({ connectorId: ModelId; connectionId: string; brandId: number; -}): Promise { - let brand = await ZendeskBrandResource.fetchByBrandId({ +}): Promise { + const syncSuccess = await syncBrandWithPermissions({ connectorId, + connectionId, brandId, + permissions: { + ticketsPermission: "none", + helpCenterPermission: "read", + }, }); - if (brand?.helpCenterPermission === "none") { - await brand.update({ helpCenterPermission: "read" }); - } - if (brand?.ticketsPermission === "none") { - await brand.update({ ticketsPermission: "read" }); - } - - const { accessToken, subdomain } = - await getZendeskSubdomainAndAccessToken(connectionId); - const zendeskApiClient = createZendeskClient({ - token: accessToken, - subdomain, - }); - - if (!brand) { - const { - result: { brand: fetchedBrand }, - } = await zendeskApiClient.brand.show(brandId); - if (fetchedBrand) { - brand = await ZendeskBrandResource.makeNew({ - blob: { - subdomain: fetchedBrand.subdomain, - connectorId: connectorId, - brandId: fetchedBrand.id, - name: fetchedBrand.name || "Brand", - helpCenterPermission: "read", - ticketsPermission: "read", - hasHelpCenter: fetchedBrand.has_help_center, - url: fetchedBrand.url, - }, - }); - } else { - logger.error({ brandId }, "[Zendesk] Brand could not be fetched."); - return null; - } + if (!syncSuccess) { + return false; // stopping early if the brand sync failed } await allowSyncZendeskHelpCenter({ @@ -71,7 +42,7 @@ export async function allowSyncZendeskBrand({ brandId, }); - return brand; + return true; } /** diff --git a/connectors/src/connectors/zendesk/lib/help_center_permissions.ts b/connectors/src/connectors/zendesk/lib/help_center_permissions.ts index b63078bee856..b074399153cf 100644 --- a/connectors/src/connectors/zendesk/lib/help_center_permissions.ts +++ b/connectors/src/connectors/zendesk/lib/help_center_permissions.ts @@ -1,5 +1,6 @@ import type { ModelId } from "@dust-tt/types"; +import { syncBrandWithPermissions } from "@connectors/connectors/zendesk/lib/utils"; import { getZendeskSubdomainAndAccessToken } from "@connectors/connectors/zendesk/lib/zendesk_access_token"; import { changeZendeskClientSubdomain, @@ -25,52 +26,31 @@ export async function allowSyncZendeskHelpCenter({ connectionId: string; brandId: number; withChildren?: boolean; -}): Promise { - let brand = await ZendeskBrandResource.fetchByBrandId({ +}): Promise { + const zendeskApiClient = createZendeskClient( + await getZendeskSubdomainAndAccessToken(connectionId) + ); + + const syncSuccess = await syncBrandWithPermissions({ + zendeskApiClient, + connectionId, connectorId, brandId, + permissions: { + ticketsPermission: "none", + helpCenterPermission: "read", + }, }); - - if (brand?.helpCenterPermission === "none") { - await brand.update({ helpCenterPermission: "read" }); - } - - const { accessToken, subdomain } = - await getZendeskSubdomainAndAccessToken(connectionId); - const zendeskApiClient = createZendeskClient({ - token: accessToken, - subdomain, - }); - - if (!brand) { - const { - result: { brand: fetchedBrand }, - } = await zendeskApiClient.brand.show(brandId); - if (fetchedBrand) { - brand = await ZendeskBrandResource.makeNew({ - blob: { - subdomain: fetchedBrand.subdomain, - connectorId: connectorId, - brandId: fetchedBrand.id, - name: fetchedBrand.name || "Brand", - helpCenterPermission: "read", - ticketsPermission: "none", - hasHelpCenter: fetchedBrand.has_help_center, - url: fetchedBrand.url, - }, - }); - } else { - logger.error( - { connectorId, brandId }, - "[Zendesk] Brand could not be fetched." - ); - return null; - } + if (!syncSuccess) { + return false; // stopping early if the brand sync failed } // updating permissions for all the children categories if (withChildren) { - await changeZendeskClientSubdomain({ client: zendeskApiClient, brandId }); + await changeZendeskClientSubdomain(zendeskApiClient, { + connectorId, + brandId, + }); try { const categories = await zendeskApiClient.helpcenter.categories.list(); categories.forEach((category) => @@ -86,11 +66,11 @@ export async function allowSyncZendeskHelpCenter({ { connectorId, brandId }, "[Zendesk] Categories could not be fetched." ); - return null; + return false; } } - return brand; + return true; } /** @@ -157,15 +137,14 @@ export async function allowSyncZendeskCategory({ await category.update({ permission: "read" }); } - const { accessToken, subdomain } = - await getZendeskSubdomainAndAccessToken(connectionId); - const zendeskApiClient = createZendeskClient({ - token: accessToken, - subdomain, - }); - if (!category) { - await changeZendeskClientSubdomain({ client: zendeskApiClient, brandId }); + const zendeskApiClient = createZendeskClient( + await getZendeskSubdomainAndAccessToken(connectionId) + ); + await changeZendeskClientSubdomain(zendeskApiClient, { + connectorId, + brandId, + }); const { result: fetchedCategory } = await zendeskApiClient.helpcenter.categories.show(categoryId); if (fetchedCategory) { diff --git a/connectors/src/connectors/zendesk/lib/permissions.ts b/connectors/src/connectors/zendesk/lib/permissions.ts index 81b72786801a..fdc18e027af4 100644 --- a/connectors/src/connectors/zendesk/lib/permissions.ts +++ b/connectors/src/connectors/zendesk/lib/permissions.ts @@ -78,38 +78,33 @@ export async function retrieveChildrenNodes({ const isReadPermissionsOnly = filterPermission === "read"; let nodes: ContentNode[] = []; - const { accessToken, subdomain } = await getZendeskSubdomainAndAccessToken( - connector.connectionId + const zendeskApiClient = createZendeskClient( + await getZendeskSubdomainAndAccessToken(connector.connectionId) ); - const zendeskApiClient = createZendeskClient({ - token: accessToken, - subdomain, - }); // At the root level, we show one node for each brand. if (!parentInternalId) { if (isReadPermissionsOnly) { - const brandsInDatabase = - await ZendeskBrandResource.fetchAllWithHelpCenter({ connectorId }); + const brandsInDatabase = await ZendeskBrandResource.fetchAllReadOnly({ + connectorId, + }); nodes = brandsInDatabase.map((brand) => brand.toContentNode({ connectorId }) ); } else { const { result: brands } = await zendeskApiClient.brand.list(); - nodes = brands - .filter((brand) => brand.has_help_center) - .map((brand) => ({ - provider: connector.type, - internalId: getBrandInternalId(connectorId, brand.id), - parentInternalId: null, - type: "folder", - title: brand.name || "Brand", - sourceUrl: brand.brand_url, - expandable: true, - permission: "none", - dustDocumentId: null, - lastUpdatedAt: null, - })); + nodes = brands.map((brand) => ({ + provider: connector.type, + internalId: getBrandInternalId(connectorId, brand.id), + parentInternalId: null, + type: "folder", + title: brand.name || "Brand", + sourceUrl: brand.brand_url, + expandable: true, + permission: "none", + dustDocumentId: null, + lastUpdatedAt: null, + })); } } else { const { type, objectId } = getIdFromInternalId( @@ -188,8 +183,8 @@ export async function retrieveChildrenNodes({ category.toContentNode({ connectorId, expandable: true }) ); } else { - await changeZendeskClientSubdomain({ - client: zendeskApiClient, + await changeZendeskClientSubdomain(zendeskApiClient, { + connectorId, brandId: objectId, }); const categories = diff --git a/connectors/src/connectors/zendesk/lib/ticket_permissions.ts b/connectors/src/connectors/zendesk/lib/ticket_permissions.ts index 19eac219a1f3..a136020617c6 100644 --- a/connectors/src/connectors/zendesk/lib/ticket_permissions.ts +++ b/connectors/src/connectors/zendesk/lib/ticket_permissions.ts @@ -1,13 +1,15 @@ import type { ModelId } from "@dust-tt/types"; -import { getZendeskSubdomainAndAccessToken } from "@connectors/connectors/zendesk/lib/zendesk_access_token"; -import { createZendeskClient } from "@connectors/connectors/zendesk/lib/zendesk_api"; +import { syncBrandWithPermissions } from "@connectors/connectors/zendesk/lib/utils"; import logger from "@connectors/logger/logger"; import { ZendeskBrandResource, ZendeskTicketResource, } from "@connectors/resources/zendesk_resources"; +/** + * Marks the node "Tickets" of a Brand as permission "read". + */ export async function allowSyncZendeskTickets({ connectorId, connectionId, @@ -16,50 +18,20 @@ export async function allowSyncZendeskTickets({ connectorId: ModelId; connectionId: string; brandId: number; -}): Promise { - let brand = await ZendeskBrandResource.fetchByBrandId({ +}): Promise { + return syncBrandWithPermissions({ connectorId, + connectionId, brandId, + permissions: { + ticketsPermission: "read", + helpCenterPermission: "none", + }, }); - if (brand?.ticketsPermission === "none") { - await brand.update({ ticketsPermission: "read" }); - } - - const { accessToken, subdomain } = - await getZendeskSubdomainAndAccessToken(connectionId); - const zendeskApiClient = createZendeskClient({ - token: accessToken, - subdomain, - }); - - if (!brand) { - const { - result: { brand: fetchedBrand }, - } = await zendeskApiClient.brand.show(brandId); - if (fetchedBrand) { - brand = await ZendeskBrandResource.makeNew({ - blob: { - subdomain: fetchedBrand.subdomain, - connectorId: connectorId, - brandId: fetchedBrand.id, - name: fetchedBrand.name || "Brand", - helpCenterPermission: "none", - ticketsPermission: "read", - hasHelpCenter: fetchedBrand.has_help_center, - url: fetchedBrand.url, - }, - }); - } else { - logger.error({ brandId }, "[Zendesk] Brand could not be fetched."); - return null; - } - } - - return brand; } /** - * Mark a help center as permission "none" and all children (collections and articles). + * Mark the node "Tickets" and all the children tickets for a Brand as permission "none". */ export async function revokeSyncZendeskTickets({ connectorId, diff --git a/connectors/src/connectors/zendesk/lib/utils.ts b/connectors/src/connectors/zendesk/lib/utils.ts new file mode 100644 index 000000000000..f7d0f217d40b --- /dev/null +++ b/connectors/src/connectors/zendesk/lib/utils.ts @@ -0,0 +1,73 @@ +import type { ModelId } from "@dust-tt/types"; +import type { Client } from "node-zendesk"; + +import { getZendeskSubdomainAndAccessToken } from "@connectors/connectors/zendesk/lib/zendesk_access_token"; +import { createZendeskClient } from "@connectors/connectors/zendesk/lib/zendesk_api"; +import logger from "@connectors/logger/logger"; +import { ZendeskBrandResource } from "@connectors/resources/zendesk_resources"; + +/** + * Syncs the permissions of a brand, fetching it and pushing it if not found in the db. + * Only works to grant permissions, to revoke permissions you would always have it in db and thus directly update. + * @returns True if the fetch succeeded, false otherwise. + */ +export async function syncBrandWithPermissions({ + zendeskApiClient = null, + connectorId, + connectionId, + brandId, + permissions, +}: { + zendeskApiClient?: Client | null; + connectorId: ModelId; + connectionId: string; + brandId: number; + permissions: { + ticketsPermission: "read" | "none"; + helpCenterPermission: "read" | "none"; + }; +}): Promise { + const brand = await ZendeskBrandResource.fetchByBrandId({ + connectorId, + brandId, + }); + + if (brand) { + if (permissions.helpCenterPermission === "read") { + await brand.grantHelpCenterPermissions(); + } + if (permissions.ticketsPermission === "read") { + await brand.grantTicketsPermissions(); + } + + return true; + } + + zendeskApiClient ||= createZendeskClient( + await getZendeskSubdomainAndAccessToken(connectionId) + ); + + const { + result: { brand: fetchedBrand }, + } = await zendeskApiClient.brand.show(brandId); + if (fetchedBrand) { + await ZendeskBrandResource.makeNew({ + blob: { + subdomain: fetchedBrand.subdomain, + connectorId: connectorId, + brandId: fetchedBrand.id, + name: fetchedBrand.name || "Brand", + ...permissions, + hasHelpCenter: fetchedBrand.has_help_center, + url: fetchedBrand.url, + }, + }); + return true; + } else { + logger.error( + { connectorId, brandId }, + "[Zendesk] Brand could not be fetched." + ); + return false; + } +} diff --git a/connectors/src/connectors/zendesk/lib/zendesk_api.ts b/connectors/src/connectors/zendesk/lib/zendesk_api.ts index 80a0b4397bad..72a85befd500 100644 --- a/connectors/src/connectors/zendesk/lib/zendesk_api.ts +++ b/connectors/src/connectors/zendesk/lib/zendesk_api.ts @@ -1,27 +1,165 @@ +import assert from "node:assert"; + +import type { ModelId } from "@dust-tt/types"; import type { Client } from "node-zendesk"; import { createClient } from "node-zendesk"; +import type { ZendeskFetchedArticle } from "@connectors/connectors/zendesk/lib/node-zendesk-types"; +import { ExternalOAuthTokenError } from "@connectors/lib/error"; +import logger from "@connectors/logger/logger"; +import { ZendeskBrandResource } from "@connectors/resources/zendesk_resources"; + +const ZENDESK_RATE_LIMIT_MAX_RETRIES = 5; +const ZENDESK_RATE_LIMIT_TIMEOUT_SECONDS = 60; + export function createZendeskClient({ - token, + accessToken, subdomain, }: { - token: string; + accessToken: string; subdomain: string; }) { - return createClient({ oauth: true, token, subdomain }); + return createClient({ oauth: true, token: accessToken, subdomain }); } -export async function changeZendeskClientSubdomain({ - client, - brandId, -}: { - client: Client; - brandId: number; -}) { - // TODO(2024-10-29 aubin): in some cases the brand in db is available and can be used to retrieve the subdomain +/** + * Returns a Zendesk client with the subdomain set to the one in the brand. + * Retrieves the brand from the database if it exists, fetches it from the Zendesk API otherwise. + */ +export async function changeZendeskClientSubdomain( + client: Client, + { connectorId, brandId }: { connectorId: ModelId; brandId: number } +) { + client.config.subdomain = await getZendeskBrandSubdomain(client, { + connectorId, + brandId, + }); + return client; +} + +/** + * Retrieves a brand's subdomain from the database if it exists, fetches it from the Zendesk API otherwise. + */ +export async function getZendeskBrandSubdomain( + client: Client, + { connectorId, brandId }: { connectorId: ModelId; brandId: number } +): Promise { + const brandInDb = await ZendeskBrandResource.fetchByBrandId({ + connectorId, + brandId, + }); + if (brandInDb) { + return brandInDb.subdomain; + } + const { result: { brand }, } = await client.brand.show(brandId); - client.config.subdomain = brand.subdomain; - return client; + return brand.subdomain; +} + +/** + * Handles rate limit responses from Zendesk API. + * Expects to find the header `Retry-After` in the response. + * https://developer.zendesk.com/api-reference/introduction/rate-limits/ + * @returns true if the rate limit was handled and the request should be retried, false otherwise. + */ +async function handleZendeskRateLimit(response: Response): Promise { + if (response.status === 429) { + const retryAfter = Math.max( + Number(response.headers.get("Retry-After")) || 1, + 1 + ); + if (retryAfter > ZENDESK_RATE_LIMIT_TIMEOUT_SECONDS) { + logger.info( + { retryAfter }, + `[Zendesk] Attempting to wait more than ${ZENDESK_RATE_LIMIT_TIMEOUT_SECONDS} s, aborting.` + ); + throw new Error( + `Zendesk retry after larger than ${ZENDESK_RATE_LIMIT_TIMEOUT_SECONDS} s, aborting.` + ); + } + logger.info( + { response, retryAfter }, + "[Zendesk] Rate limit hit, waiting before retrying." + ); + await new Promise((resolve) => setTimeout(resolve, retryAfter * 1000)); + return true; + } + return false; +} + +/** + * Fetches a batch of articles in a category from the Zendesk API. + */ +export async function fetchZendeskArticlesInCategory({ + subdomain, + accessToken, + categoryId, + pageSize = 100, + cursor = null, +}: { + subdomain: string; + accessToken: string; + categoryId: number; + pageSize?: number; + cursor?: string | null; +}): Promise<{ + articles: ZendeskFetchedArticle[]; + meta: { has_more: boolean; after_cursor: string }; +}> { + assert( + pageSize <= 100, + `pageSize must be at most 100 (current value: ${pageSize})` // https://developer.zendesk.com/api-reference/introduction/pagination + ); + const runFetch = async () => + fetch( + `https://${subdomain}.zendesk.com/api/v2/help_center/categories/${categoryId}/articles?page[size]=${pageSize}` + + (cursor ? `&page[after]=${cursor}` : ""), + { + method: "GET", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + } + ); + + let rawResponse = await runFetch(); + + let retryCount = 0; + while (await handleZendeskRateLimit(rawResponse)) { + rawResponse = await runFetch(); + retryCount++; + if (retryCount >= ZENDESK_RATE_LIMIT_MAX_RETRIES) { + logger.info( + { response: rawResponse }, + `[Zendesk] Rate limit hit more than ${ZENDESK_RATE_LIMIT_MAX_RETRIES}, aborting.` + ); + throw new Error( + `Zendesk rate limit hit more than ${ZENDESK_RATE_LIMIT_MAX_RETRIES} times, aborting.` + ); + } + } + + const text = await rawResponse.text(); + const response = JSON.parse(text); + + if (!rawResponse.ok) { + if ( + response.type === "error.list" && + response.errors && + response.errors.length > 0 + ) { + const error = response.errors[0]; + if (error.code === "unauthorized") { + throw new ExternalOAuthTokenError(); + } + if (error.code === "not_found") { + return { articles: [], meta: { has_more: false, after_cursor: "" } }; + } + } + } + + return response; } diff --git a/connectors/src/connectors/zendesk/temporal/activities.ts b/connectors/src/connectors/zendesk/temporal/activities.ts index 5a1ac14c1075..9aac51a909af 100644 --- a/connectors/src/connectors/zendesk/temporal/activities.ts +++ b/connectors/src/connectors/zendesk/temporal/activities.ts @@ -8,6 +8,8 @@ import { getZendeskSubdomainAndAccessToken } from "@connectors/connectors/zendes import { changeZendeskClientSubdomain, createZendeskClient, + fetchZendeskArticlesInCategory, + getZendeskBrandSubdomain, } from "@connectors/connectors/zendesk/lib/zendesk_api"; import { syncArticle } from "@connectors/connectors/zendesk/temporal/sync_article"; import { syncTicket } from "@connectors/connectors/zendesk/temporal/sync_ticket"; @@ -24,6 +26,8 @@ import { } from "@connectors/resources/zendesk_resources"; import type { DataSourceConfig } from "@connectors/types/data_source_config"; +const ZENDESK_ARTICLE_BATCH_SIZE = 100; + async function _getZendeskConnectorOrRaise(connectorId: ModelId) { const connector = await ConnectorResource.fetchById(connectorId); if (!connector) { @@ -122,13 +126,9 @@ export async function syncZendeskBrandActivity({ return { helpCenterAllowed: false, ticketsAllowed: false }; } - const { accessToken, subdomain } = await getZendeskSubdomainAndAccessToken( - connector.connectionId + const zendeskApiClient = createZendeskClient( + await getZendeskSubdomainAndAccessToken(connector.connectionId) ); - const zendeskApiClient = createZendeskClient({ - token: accessToken, - subdomain, - }); // if the brand is not on Zendesk anymore, we delete it const { @@ -239,14 +239,10 @@ export async function getZendeskCategoriesActivity({ brandId: number; }): Promise { const connector = await _getZendeskConnectorOrRaise(connectorId); - const { accessToken, subdomain } = await getZendeskSubdomainAndAccessToken( - connector.connectionId + const client = createZendeskClient( + await getZendeskSubdomainAndAccessToken(connector.connectionId) ); - const client = createZendeskClient({ - token: accessToken, - subdomain, - }); - await changeZendeskClientSubdomain({ client, brandId }); + await changeZendeskClientSubdomain(client, { connectorId, brandId }); const categories = await client.helpcenter.categories.list(); return categories.map((category) => category.id); @@ -272,7 +268,7 @@ export async function syncZendeskCategoryActivity({ categoryId: number; brandId: number; currentSyncDateMs: number; -}): Promise { +}): Promise { const connector = await _getZendeskConnectorOrRaise(connectorId); const dataSourceConfig = dataSourceConfigFromConnector(connector); const categoryInDb = await _getZendeskCategoryOrRaise({ @@ -284,17 +280,16 @@ export async function syncZendeskCategoryActivity({ if (categoryInDb.permission === "none") { await deleteCategoryChildren({ connectorId, dataSourceConfig, categoryId }); await categoryInDb.delete(); - return null; + return false; } - const { accessToken, subdomain } = await getZendeskSubdomainAndAccessToken( - connector.connectionId + const zendeskApiClient = createZendeskClient( + await getZendeskSubdomainAndAccessToken(connector.connectionId) ); - const zendeskApiClient = createZendeskClient({ - token: accessToken, - subdomain, + await changeZendeskClientSubdomain(zendeskApiClient, { + connectorId, + brandId, }); - await changeZendeskClientSubdomain({ client: zendeskApiClient, brandId }); // if the category is not on Zendesk anymore, we delete it const { result: fetchedCategory } = @@ -302,7 +297,7 @@ export async function syncZendeskCategoryActivity({ if (!fetchedCategory) { await deleteCategoryChildren({ connectorId, categoryId, dataSourceConfig }); await categoryInDb.delete(); - return null; + return false; } // otherwise, we update the category name and lastUpsertedTs @@ -310,24 +305,26 @@ export async function syncZendeskCategoryActivity({ name: fetchedCategory.name || "Category", lastUpsertedTs: new Date(currentSyncDateMs), }); - return categoryInDb; + return true; } /** - * This activity is responsible for syncing all the articles in a Category. + * This activity is responsible for syncing the next batch of articles to process. * It does not sync the Category, only the Articles. */ -export async function syncZendeskArticlesActivity({ +export async function syncZendeskArticleBatchActivity({ connectorId, - category, + categoryId, currentSyncDateMs, forceResync, + cursor, }: { connectorId: ModelId; - category: ZendeskCategoryResource; + categoryId: number; currentSyncDateMs: number; forceResync: boolean; -}) { + cursor?: string | null; +}): Promise<{ hasMore: boolean; afterCursor: string | null }> { const connector = await _getZendeskConnectorOrRaise(connectorId); const dataSourceConfig = dataSourceConfigFromConnector(connector); const loggerArgs = { @@ -336,18 +333,30 @@ export async function syncZendeskArticlesActivity({ provider: "zendesk", dataSourceId: dataSourceConfig.dataSourceId, }; + const category = await _getZendeskCategoryOrRaise({ + connectorId, + categoryId, + }); const { accessToken, subdomain } = await getZendeskSubdomainAndAccessToken( connector.connectionId ); - const zendeskApiClient = createZendeskClient({ - token: accessToken, - subdomain: subdomain, + const zendeskApiClient = createZendeskClient({ accessToken, subdomain }); + const brandSubdomain = await getZendeskBrandSubdomain(zendeskApiClient, { + brandId: category.brandId, + connectorId, }); - const articles = await zendeskApiClient.helpcenter.articles.listByCategory( - category.categoryId - ); + const { + articles, + meta: { after_cursor, has_more }, + } = await fetchZendeskArticlesInCategory({ + subdomain: brandSubdomain, + accessToken, + categoryId: category.categoryId, + pageSize: ZENDESK_ARTICLE_BATCH_SIZE, + cursor, + }); await concurrentExecutor( articles, @@ -363,6 +372,7 @@ export async function syncZendeskArticlesActivity({ }), { concurrency: 10 } ); + return { hasMore: has_more, afterCursor: after_cursor }; } /** @@ -388,14 +398,13 @@ export async function syncZendeskTicketsActivity({ dataSourceId: dataSourceConfig.dataSourceId, }; - const { accessToken, subdomain } = await getZendeskSubdomainAndAccessToken( - connector.connectionId + const zendeskApiClient = createZendeskClient( + await getZendeskSubdomainAndAccessToken(connector.connectionId) ); - const zendeskApiClient = createZendeskClient({ - token: accessToken, - subdomain, + await changeZendeskClientSubdomain(zendeskApiClient, { + connectorId, + brandId, }); - await changeZendeskClientSubdomain({ client: zendeskApiClient, brandId }); const tickets = await zendeskApiClient.tickets.list(); await concurrentExecutor( diff --git a/connectors/src/connectors/zendesk/temporal/config.ts b/connectors/src/connectors/zendesk/temporal/config.ts index e43a7b45b9db..59c1a9cbbb4d 100644 --- a/connectors/src/connectors/zendesk/temporal/config.ts +++ b/connectors/src/connectors/zendesk/temporal/config.ts @@ -1,2 +1,2 @@ -export const WORKFLOW_VERSION = 2; +export const WORKFLOW_VERSION = 3; export const QUEUE_NAME = `zendesk-queue-v${WORKFLOW_VERSION}`; diff --git a/connectors/src/connectors/zendesk/temporal/sync_article.ts b/connectors/src/connectors/zendesk/temporal/sync_article.ts index 26bf7add48d2..4ab435202bfe 100644 --- a/connectors/src/connectors/zendesk/temporal/sync_article.ts +++ b/connectors/src/connectors/zendesk/temporal/sync_article.ts @@ -15,6 +15,9 @@ import type { DataSourceConfig } from "@connectors/types/data_source_config"; const turndownService = new TurndownService(); +/** + * Syncs an article from Zendesk to the postgres db and to the data sources. + */ export async function syncArticle({ connectorId, article, @@ -36,64 +39,51 @@ export async function syncArticle({ connectorId, articleId: article.id, }); - const createdAtDate = new Date(article.created_at); const updatedAtDate = new Date(article.updated_at); - const articleUpdatedAtDate = new Date(article.updated_at); - const shouldPerformUpsertion = forceResync || !articleInDb || !articleInDb.lastUpsertedTs || - articleInDb.lastUpsertedTs < articleUpdatedAtDate; // upserting if the article was updated after the last upsert + articleInDb.lastUpsertedTs < updatedAtDate; // upserting if the article was updated after the last upsert + const updatableFields = { + createdAt: new Date(article.created_at), + updatedAt: updatedAtDate, + categoryId: category.categoryId, // an article can be moved from one category to another, which does not apply to brands + name: article.name, + url: article.html_url, + }; + // we either create a new article or update the existing one if (!articleInDb) { articleInDb = await ZendeskArticleResource.makeNew({ blob: { - createdAt: createdAtDate, - updatedAt: updatedAtDate, + ...updatableFields, articleId: article.id, brandId: category.brandId, - categoryId: category.categoryId, permission: "read", - name: article.name, - url: article.html_url, connectorId, }, }); } else { - await articleInDb.update({ - createdAt: createdAtDate, - updatedAt: updatedAtDate, - categoryId: category.categoryId, // an article can be moved from one category to another, which does not apply to brands - name: article.name, - url: article.url, - }); + await articleInDb.update(updatableFields); } + logger.info( + { + ...loggerArgs, + connectorId, + articleId: article.id, + articleUpdatedAt: updatedAtDate, + dataSourceLastUpsertedAt: articleInDb?.lastUpsertedTs ?? null, + }, + shouldPerformUpsertion + ? "[Zendesk] Article to sync." + : "[Zendesk] Article already up to date. Skipping sync." + ); + if (!shouldPerformUpsertion) { - logger.info( - { - ...loggerArgs, - connectorId, - articleId: article.id, - articleUpdatedAt: articleUpdatedAtDate, - dataSourceLastUpsertedAt: articleInDb?.lastUpsertedTs ?? null, - }, - "[Zendesk] Article already up to date. Skipping sync." - ); return; - } else { - logger.info( - { - ...loggerArgs, - connectorId, - articleId: article.id, - articleUpdatedAt: articleUpdatedAtDate, - dataSourceLastUpsertedAt: articleInDb?.lastUpsertedTs ?? null, - }, - "[Zendesk] Article to sync." - ); } const categoryContent = diff --git a/connectors/src/connectors/zendesk/temporal/workflows.ts b/connectors/src/connectors/zendesk/temporal/workflows.ts index 73eae4036d36..d82436785dbb 100644 --- a/connectors/src/connectors/zendesk/temporal/workflows.ts +++ b/connectors/src/connectors/zendesk/temporal/workflows.ts @@ -13,13 +13,12 @@ import type { ZendeskUpdateSignal, } from "@connectors/connectors/zendesk/temporal/signals"; import { zendeskUpdatesSignal } from "@connectors/connectors/zendesk/temporal/signals"; -import type { ZendeskCategoryResource } from "@connectors/resources/zendesk_resources"; const { getZendeskCategoriesActivity, syncZendeskBrandActivity, syncZendeskCategoryActivity, - syncZendeskArticlesActivity, + syncZendeskArticleBatchActivity, syncZendeskTicketsActivity, } = proxyActivities({ startToCloseTimeout: "5 minutes", @@ -317,21 +316,30 @@ export async function zendeskCategorySyncWorkflow({ currentSyncDateMs: number; forceResync: boolean; }) { - const category = await syncZendeskCategoryActivity({ + const wasCategoryUpdated = await syncZendeskCategoryActivity({ connectorId, categoryId, currentSyncDateMs, brandId, }); - if (!category) { + if (!wasCategoryUpdated) { return; // nothing to sync } - await syncZendeskArticlesActivity({ - connectorId, - category, - currentSyncDateMs, - forceResync, - }); + + let cursor = null; // cursor involved in the pagination of the API + let hasMore = true; + + while (hasMore) { + const result = await syncZendeskArticleBatchActivity({ + connectorId, + categoryId, + currentSyncDateMs, + forceResync, + cursor, + }); + hasMore = result.hasMore || false; + cursor = result.afterCursor; + } } /** @@ -352,27 +360,34 @@ async function runZendeskBrandHelpCenterSyncActivities({ connectorId, brandId, }); - const categoriesToSync = new Set(); + const categoryIdsToSync = new Set(); for (const categoryId of categoryIds) { - const category = await syncZendeskCategoryActivity({ + const wasCategoryUpdated = await syncZendeskCategoryActivity({ connectorId, categoryId, currentSyncDateMs, brandId, }); - if (category) { - categoriesToSync.add(category); + if (wasCategoryUpdated) { + categoryIdsToSync.add(categoryId); } } - /// grouping the articles by category for a lower granularity - for (const category of categoriesToSync) { - await syncZendeskArticlesActivity({ - connectorId, - category, - currentSyncDateMs, - forceResync, - }); + for (const categoryId of categoryIdsToSync) { + let hasMore = true; + let cursor = null; // cursor involved in the pagination of the API + + while (hasMore) { + const result = await syncZendeskArticleBatchActivity({ + connectorId, + categoryId, + currentSyncDateMs, + forceResync, + cursor, + }); + hasMore = result.hasMore || false; + cursor = result.afterCursor; + } } } diff --git a/connectors/src/resources/zendesk_resources.ts b/connectors/src/resources/zendesk_resources.ts index 41115fa24809..f939d93fd354 100644 --- a/connectors/src/resources/zendesk_resources.ts +++ b/connectors/src/resources/zendesk_resources.ts @@ -164,6 +164,18 @@ export class ZendeskBrandResource extends BaseResource { return new this(this.model, brand.get()); } + async grantHelpCenterPermissions(): Promise { + if (this.helpCenterPermission === "none") { + await this.update({ helpCenterPermission: "read" }); + } + } + + async grantTicketsPermissions(): Promise { + if (this.ticketsPermission === "none") { + await this.update({ ticketsPermission: "read" }); + } + } + async revokeAllPermissions(): Promise { await this.revokeHelpCenterPermissions(); await this.revokeTicketsPermissions();