From e108613f8221259ac6715c57b2d16d7e6f356a7b Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Tue, 31 Dec 2024 14:56:18 +0100 Subject: [PATCH 01/23] Webcrawler: bump heartbeat timeout (#9675) --- .../src/connectors/webcrawler/temporal/workflows.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/connectors/src/connectors/webcrawler/temporal/workflows.ts b/connectors/src/connectors/webcrawler/temporal/workflows.ts index 510db387cb31..5542224947c2 100644 --- a/connectors/src/connectors/webcrawler/temporal/workflows.ts +++ b/connectors/src/connectors/webcrawler/temporal/workflows.ts @@ -13,21 +13,19 @@ import type * as activities from "@connectors/connectors/webcrawler/temporal/act // timeout for crawling a single url = timeout for upserting (5 minutes) + 2mn // leeway to crawl on slow websites export const REQUEST_HANDLING_TIMEOUT = 420; +// For each page crawl we have an heartbeat but some crawls seem to stall for longer periods. Giving +// them 20mn to hearbeat. +export const HEARTBEAT_TIMEOUT = 1200; export const MAX_TIME_TO_CRAWL_MINUTES = 240; - export const MIN_EXTRACTED_TEXT_LENGTH = 1024; - export const MAX_BLOCKED_RATIO = 0.9; - export const MAX_PAGES_TOO_LARGE_RATIO = 0.9; const { crawlWebsiteByConnectorId, webCrawlerGarbageCollector } = proxyActivities({ startToCloseTimeout: `${MAX_TIME_TO_CRAWL_MINUTES} minutes`, - // for each page crawl, there are heartbeats, but a page crawl can last at max - // REQUEST_HANDLING_TIMEOUT seconds - heartbeatTimeout: `${REQUEST_HANDLING_TIMEOUT + 120} seconds`, + heartbeatTimeout: `${HEARTBEAT_TIMEOUT} seconds`, cancellationType: ActivityCancellationType.TRY_CANCEL, retry: { initialInterval: `${REQUEST_HANDLING_TIMEOUT * 2} seconds`, From feb3737fd47f98a006d9448c2b17d234b9bfb9e3 Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Tue, 31 Dec 2024 14:57:31 +0100 Subject: [PATCH 02/23] poke delete: delete feature flags before workspace (#9676) --- front/poke/temporal/activities.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/front/poke/temporal/activities.ts b/front/poke/temporal/activities.ts index 639f7f4ed6b8..7778fc3309b0 100644 --- a/front/poke/temporal/activities.ts +++ b/front/poke/temporal/activities.ts @@ -38,6 +38,7 @@ import { AgentUserRelation, GlobalAgentSettings, } from "@app/lib/models/assistant/agent"; +import { FeatureFlag } from "@app/lib/models/feature_flag"; import { Subscription } from "@app/lib/models/plan"; import { DustAppSecret, @@ -536,6 +537,11 @@ export async function deleteWorkspaceActivity({ workspaceId: workspace.id, }, }); + await FeatureFlag.destroy({ + where: { + workspaceId: workspace.id, + }, + }); hardDeleteLogger.info({ workspaceId }, "Deleting Workspace"); From 8e7ab75490ca59cb3ecccca4a2dedf776139d2a9 Mon Sep 17 00:00:00 2001 From: thib-martin <168569391+thib-martin@users.noreply.github.com> Date: Tue, 31 Dec 2024 16:02:28 +0100 Subject: [PATCH 03/23] adding workspace health link in poke (#9671) * adding workspace health link * fixing linting * import link * lint * fixing bug * Ran linter --------- Co-authored-by: Lucas --- front/components/poke/workspace/table.tsx | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/front/components/poke/workspace/table.tsx b/front/components/poke/workspace/table.tsx index 4cf2acbf4c2c..632d591d413b 100644 --- a/front/components/poke/workspace/table.tsx +++ b/front/components/poke/workspace/table.tsx @@ -3,6 +3,7 @@ import type { WorkspaceDomain, WorkspaceType, } from "@dust-tt/types"; +import Link from "next/link"; import { PokeTable, @@ -39,6 +40,18 @@ export function WorkspaceInfoTable({ sId + + Workspace Health + + + Metabase + + + Creation {worspaceCreationDay} From 1e414436b67b5f6b1a47d7ac1411c401a3ace9d7 Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Tue, 31 Dec 2024 16:24:46 +0100 Subject: [PATCH 04/23] poke delete: handle deleted spaces (#9677) --- front/lib/resources/space_resource.ts | 6 ++++-- front/poke/temporal/activities.ts | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/front/lib/resources/space_resource.ts b/front/lib/resources/space_resource.ts index 0543f9e3b4d2..133abd694e12 100644 --- a/front/lib/resources/space_resource.ts +++ b/front/lib/resources/space_resource.ts @@ -182,9 +182,11 @@ export class SpaceResource extends BaseResource { static async listWorkspaceSpaces( auth: Authenticator, - options?: { includeConversationsSpace?: boolean } + options?: { includeConversationsSpace?: boolean; includeDeleted?: boolean } ): Promise { - const spaces = await this.baseFetch(auth); + const spaces = await this.baseFetch(auth, { + includeDeleted: options?.includeDeleted, + }); if (!options?.includeConversationsSpace) { return spaces.filter((s) => !s.isConversations()); diff --git a/front/poke/temporal/activities.ts b/front/poke/temporal/activities.ts index 7778fc3309b0..4454164b92fc 100644 --- a/front/poke/temporal/activities.ts +++ b/front/poke/temporal/activities.ts @@ -494,6 +494,7 @@ export async function deleteSpacesActivity({ const auth = await Authenticator.internalAdminForWorkspace(workspaceId); const spaces = await SpaceResource.listWorkspaceSpaces(auth, { includeConversationsSpace: true, + includeDeleted: true, }); for (const space of spaces) { From 4cd69c3b9d05305b5ae02cde3c432ac841337a82 Mon Sep 17 00:00:00 2001 From: Flavien David Date: Tue, 31 Dec 2024 16:49:24 +0100 Subject: [PATCH 05/23] Flav/improve confluence workflow (#9678) * Improve fetching root pages * Improve workflow history size. --- .../confluence/temporal/activities.ts | 45 ++++++- .../confluence/temporal/workflows.ts | 125 +++++++++--------- 2 files changed, 106 insertions(+), 64 deletions(-) diff --git a/connectors/src/connectors/confluence/temporal/activities.ts b/connectors/src/connectors/confluence/temporal/activities.ts index 0e6b09698093..5819132abc4d 100644 --- a/connectors/src/connectors/confluence/temporal/activities.ts +++ b/connectors/src/connectors/confluence/temporal/activities.ts @@ -607,7 +607,7 @@ export async function confluenceGetActiveChildPageRefsActivity({ // Confluence has a single main landing page. // However, users have the ability to create "orphaned" root pages that don't link from the main landing. // It's important to ensure these pages are also imported. -export async function confluenceGetRootPageRefsActivity({ +async function getRootPageRefsActivity({ connectorId, confluenceCloudId, spaceId, @@ -647,6 +647,49 @@ export async function confluenceGetRootPageRefsActivity({ } } +// Activity to handle fetching, upserting, and filtering root pages. +export async function fetchAndUpsertRootPagesActivity(params: { + confluenceCloudId: string; + connectorId: ModelId; + forceUpsert: boolean; + isBatchSync: boolean; + spaceId: string; + spaceName: string; + visitedAtMs: number; +}): Promise { + const { connectorId, confluenceCloudId, spaceId } = params; + + // Get the root level pages for the space. + const rootPageRefs = await getRootPageRefsActivity({ + connectorId, + confluenceCloudId, + spaceId, + }); + if (rootPageRefs.length === 0) { + return []; + } + + const allowedRootPageIds: string[] = []; + + // Check and upsert pages, filter allowed ones. + for (const rootPageRef of rootPageRefs) { + const successfullyUpsert = await confluenceCheckAndUpsertPageActivity({ + ...params, + pageRef: rootPageRef, + }); + + // If the page fails the upsert operation, it indicates the page is restricted. + // Such pages should not be added to the list of allowed pages. + if (successfullyUpsert) { + allowedRootPageIds.push(rootPageRef.id); + } + } + + console.log(">> allowedRootPageIds", allowedRootPageIds); + + return allowedRootPageIds; +} + export async function confluenceGetTopLevelPageIdsActivity({ confluenceCloudId, connectorId, diff --git a/connectors/src/connectors/confluence/temporal/workflows.ts b/connectors/src/connectors/confluence/temporal/workflows.ts index a0ef599c9f01..b421e473e2c4 100644 --- a/connectors/src/connectors/confluence/temporal/workflows.ts +++ b/connectors/src/connectors/confluence/temporal/workflows.ts @@ -29,7 +29,6 @@ const { confluenceUpdatePagesParentIdsActivity, confluenceCheckAndUpsertPageActivity, confluenceGetActiveChildPageRefsActivity, - confluenceGetRootPageRefsActivity, fetchConfluenceSpaceIdsForConnectorActivity, confluenceUpsertPageWithFullParentsActivity, @@ -38,6 +37,9 @@ const { fetchConfluenceConfigurationActivity, confluenceUpsertSpaceFolderActivity, + + fetchAndUpsertRootPagesActivity, + getSpaceIdsToSyncActivity, } = proxyActivities({ startToCloseTimeout: "30 minutes", @@ -51,6 +53,7 @@ const { // avoid exceeding Temporal's max workflow size limit, // since a Confluence page can have an unbounded number of pages. const TEMPORAL_WORKFLOW_MAX_HISTORY_LENGTH = 10_000; +const TEMPORAL_WORKFLOW_MAX_HISTORY_SIZE_MB = 10; export async function confluenceSyncWorkflow({ connectorId, @@ -161,39 +164,16 @@ export async function confluenceSpaceSyncWorkflow( spaceName, }); - // Get the root level pages for the space. - const rootPageRefs = await confluenceGetRootPageRefsActivity({ - connectorId, + const allowedRootPageIds = await fetchAndUpsertRootPagesActivity({ + ...params, confluenceCloudId, - spaceId, + spaceName, + visitedAtMs, }); - if (rootPageRefs.length === 0) { - return; - } - - const allowedRootPageRefs = new Map( - rootPageRefs.map((r) => [r.id, r]) - ); - - // Upsert the root pages. - for (const rootPageRef of allowedRootPageRefs.values()) { - const successfullyUpsert = await confluenceCheckAndUpsertPageActivity({ - ...params, - spaceName, - pageRef: rootPageRef, - visitedAtMs, - }); - - // If the page fails the upsert operation, it indicates the page is restricted. - // Such pages should be excluded from the list of allowed pages. - if (!successfullyUpsert) { - allowedRootPageRefs.delete(rootPageRef.id); - } - } // Fetch all top-level pages within a specified space. Top-level pages // refer to those directly nested under the space's root pages. - for (const allowedRootPageId of allowedRootPageRefs.keys()) { + for (const allowedRootPageId of allowedRootPageIds) { let nextPageCursor: string | null = ""; do { const { topLevelPageRefs, nextPageCursor: nextCursor } = @@ -246,6 +226,8 @@ export async function confluenceSpaceSyncWorkflow( ); } +type StackElement = ConfluencePageRef | { parentId: string; cursor: string }; + interface confluenceSyncTopLevelChildPagesWorkflowInput { confluenceCloudId: string; connectorId: ModelId; @@ -253,57 +235,74 @@ interface confluenceSyncTopLevelChildPagesWorkflowInput { isBatchSync: boolean; spaceId: string; spaceName: string; - topLevelPageRefs: ConfluencePageRef[]; + topLevelPageRefs: StackElement[]; visitedAtMs: number; } -// This Workflow implements a DFS algorithm to synchronize all pages not -// subject to restrictions. It stops importing child pages -// if a parent page is restricted. -// Page restriction checks are performed by `confluenceCheckAndUpsertPageActivity`; -// where false denotes restriction. Children of unrestricted pages are -// stacked for subsequent import. +/** + * This workflow implements a DFS algorithm to synchronize all pages not subject to restrictions. + * It uses a stack to process pages and their children, with a special handling for pagination: + * - Regular pages are processed and their children are added to the stack + * - Cursor elements in the stack represent continuation points for pages with many children + * This ensures we never store too many pages in the workflow history while maintaining proper + * traversal. + * + * The workflow stops importing child pages if a parent page is restricted. + * Page restriction checks are performed by `confluenceCheckAndUpsertPageActivity`. + */ export async function confluenceSyncTopLevelChildPagesWorkflow( params: confluenceSyncTopLevelChildPagesWorkflowInput ) { const { spaceName, topLevelPageRefs, visitedAtMs } = params; - const stack = [...topLevelPageRefs]; + const stack: StackElement[] = [...topLevelPageRefs]; while (stack.length > 0) { - const currentPageRef = stack.pop(); - if (!currentPageRef) { + const current = stack.pop(); + if (!current) { throw new Error("No more pages to parse."); } - const successfullyUpsert = await confluenceCheckAndUpsertPageActivity({ - ...params, - spaceName, - pageRef: currentPageRef, - visitedAtMs, - }); - if (!successfullyUpsert) { - continue; - } + // Check if it's a page reference or cursor. + const isPageRef = "id" in current; - // Fetch child pages of the current top level page. - let nextPageCursor: string | null = ""; - do { - const { childPageRefs, nextPageCursor: nextCursor } = - await confluenceGetActiveChildPageRefsActivity({ - ...params, - parentPageId: currentPageRef.id, - pageCursor: nextPageCursor, - }); + // If it's a page, process it first. + if (isPageRef) { + const successfullyUpsert = await confluenceCheckAndUpsertPageActivity({ + ...params, + spaceName, + pageRef: current, + visitedAtMs, + }); + if (!successfullyUpsert) { + continue; + } + } - nextPageCursor = nextCursor; // Prepare for the next iteration. + // Get child pages using either initial empty cursor or saved cursor. + const { childPageRefs, nextPageCursor } = + await confluenceGetActiveChildPageRefsActivity({ + ...params, + parentPageId: isPageRef ? current.id : current.parentId, + pageCursor: isPageRef ? "" : current.cursor, + }); - stack.push(...childPageRefs); - } while (nextPageCursor !== null); + // Add children and next cursor if there are more. + stack.push(...childPageRefs); + if (nextPageCursor !== null) { + stack.push({ + parentId: isPageRef ? current.id : current.parentId, + cursor: nextPageCursor, + }); + } - // If additional pages are pending and workflow limits are reached, continue in a new workflow. + // Check if we would exceed limits by continuing. + const hasReachedWorkflowLimits = + workflowInfo().historyLength > TEMPORAL_WORKFLOW_MAX_HISTORY_LENGTH || + workflowInfo().historySize > + TEMPORAL_WORKFLOW_MAX_HISTORY_SIZE_MB * 1024 * 1024; if ( - stack.length > 0 && - workflowInfo().historyLength > TEMPORAL_WORKFLOW_MAX_HISTORY_LENGTH + hasReachedWorkflowLimits && + (stack.length > 0 || childPageRefs.length > 0 || nextPageCursor !== null) ) { await continueAsNew({ ...params, From 3acc96efb54897c4779cf7bc7a819b544ea598ad Mon Sep 17 00:00:00 2001 From: Flavien David Date: Wed, 1 Jan 2025 18:34:06 +0100 Subject: [PATCH 06/23] Optimize Confluence sync performance and rate limit handling (#9681) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Only check page restrictions every X hours. * Improve retry on Confluence API rate limit * ✨ * 🔙 * Fix DD instrumentation * ✨ * ✨ * Remove workflow error from client abstraction * 📖 * 🔙 --- .../confluence/lib/confluence_client.ts | 77 ++++++++++++------- .../confluence/temporal/cast_known_errors.ts | 8 ++ .../confluence/temporal/workflows.ts | 1 + 3 files changed, 58 insertions(+), 28 deletions(-) diff --git a/connectors/src/connectors/confluence/lib/confluence_client.ts b/connectors/src/connectors/confluence/lib/confluence_client.ts index df4ba53370d7..c3f1b8cc7b05 100644 --- a/connectors/src/connectors/confluence/lib/confluence_client.ts +++ b/connectors/src/connectors/confluence/lib/confluence_client.ts @@ -3,10 +3,7 @@ import { isLeft } from "fp-ts/Either"; import * as t from "io-ts"; import { setTimeoutAsync } from "@connectors/lib/async_utils"; -import { - ExternalOAuthTokenError, - ProviderWorkflowError, -} from "@connectors/lib/error"; +import { ExternalOAuthTokenError } from "@connectors/lib/error"; import logger from "@connectors/logger/logger"; import { statsDClient } from "@connectors/logger/withlogging"; @@ -147,10 +144,11 @@ const ConfluenceReadOperationRestrictionsCodec = t.type({ restrictions: RestrictionsCodec, }); -// default number of ms we wait before retrying after a rate limit hit. -const DEFAULT_RETRY_AFTER_DURATION_MS = 10 * 1000; -// Number of times we retry when rate limited (429). -const MAX_RATE_LIMIT_RETRY_COUNT = 10; +// If Confluence does not provide a retry-after header, we use this constant to signal no delay. +const NO_RETRY_AFTER_DELAY = -1; +// Number of times we retry when rate limited and Confluence does provide a retry-after header. +const MAX_RATE_LIMIT_RETRY_COUNT = 5; + // Space types that we support indexing in Dust. export const CONFLUENCE_SUPPORTED_SPACE_TYPES = [ "global", @@ -173,11 +171,11 @@ function getRetryAfterDuration(response: Response): number { const retryAfter = response.headers.get("retry-after"); // https://developer.atlassian.com/cloud/confluence/rate-limiting/ if (retryAfter) { const delay = parseInt(retryAfter, 10); - return !Number.isNaN(delay) - ? delay * 1000 - : DEFAULT_RETRY_AFTER_DURATION_MS; + + return !Number.isNaN(delay) ? delay * 1000 : NO_RETRY_AFTER_DELAY; } - return DEFAULT_RETRY_AFTER_DURATION_MS; + + return NO_RETRY_AFTER_DELAY; } export class ConfluenceClient { @@ -244,38 +242,61 @@ export class ConfluenceClient { })(); if (!response.ok) { - statsDClient.increment("external.api.calls", 1, [ - "provider:confluence", - "status:error", - ]); - // If the token is invalid, the API will return a 403 Forbidden response. if (response.status === 403 && response.statusText === "Forbidden") { throw new ExternalOAuthTokenError(); } - // retry the request after a delay: https://developer.atlassian.com/cloud/confluence/rate-limiting/ + + // Handle rate limiting from Confluence API + // https://developer.atlassian.com/cloud/confluence/rate-limiting/ + // + // Current strategy: + // 1. If Confluence provides a retry-after header, we honor it immediately + // by sleeping in the client. This is not ideal but provides the most + // accurate rate limit handling until we can use Temporal's nextRetryDelay. + // 2. If no retry-after header is provided, we throw a transient error and + // let Temporal handle the retry with exponential backoff. + // + // Once we upgrade to Temporal SDK >= X.Y.Z, we should: + // - Remove the client-side sleep + // - Use ApplicationFailure.create() with nextRetryDelay + // - See: https://docs.temporal.io/develop/typescript/failure-detection#activity-next-retry-delay if (response.status === 429) { + statsDClient.increment("external.api.calls", 1, [ + "provider:confluence", + "status:rate_limited", + ]); + if (retryCount < MAX_RATE_LIMIT_RETRY_COUNT) { const delayMs = getRetryAfterDuration(response); logger.warn( { endpoint, - retryCount, delayMs, }, - "[Confluence] Rate limit hit, retrying after delay" - ); - await setTimeoutAsync(delayMs); - return this.request(endpoint, codec, retryCount + 1); - } else { - throw new ProviderWorkflowError( - "confluence", - "Rate limit hit on confluence API more than 10 times.", - "rate_limit_error" + "[Confluence] Rate limit hit" ); + + // Only retry rate-limited requests when the server provides a Retry-After delay. + if (delayMs !== NO_RETRY_AFTER_DELAY) { + await setTimeoutAsync(delayMs); + return this.request(endpoint, codec, retryCount + 1); + } } + + // Otherwise throw regular error to let downstream handle retries (e.g: Temporal). + throw new ConfluenceClientError("Confluence API rate limit exceeded", { + type: "http_response_error", + status: response.status, + data: { url: `${this.apiUrl}${endpoint}`, response }, + }); } + statsDClient.increment("external.api.calls", 1, [ + "provider:confluence", + "status:error", + ]); + throw new ConfluenceClientError( `Confluence API responded with status: ${response.status}: ${this.apiUrl}${endpoint}`, { diff --git a/connectors/src/connectors/confluence/temporal/cast_known_errors.ts b/connectors/src/connectors/confluence/temporal/cast_known_errors.ts index 6dd225ebc311..5b89236802e2 100644 --- a/connectors/src/connectors/confluence/temporal/cast_known_errors.ts +++ b/connectors/src/connectors/confluence/temporal/cast_known_errors.ts @@ -22,6 +22,14 @@ export class ConfluenceCastKnownErrorsInterceptor err.type === "http_response_error" ) { switch (err.status) { + case 429: + throw new ProviderWorkflowError( + "confluence", + "429 - Rate Limit Exceeded", + "rate_limit_error", + err + ); + case 500: throw new ProviderWorkflowError( "confluence", diff --git a/connectors/src/connectors/confluence/temporal/workflows.ts b/connectors/src/connectors/confluence/temporal/workflows.ts index b421e473e2c4..3074cd740915 100644 --- a/connectors/src/connectors/confluence/temporal/workflows.ts +++ b/connectors/src/connectors/confluence/temporal/workflows.ts @@ -45,6 +45,7 @@ const { startToCloseTimeout: "30 minutes", retry: { initialInterval: "60 seconds", + backoffCoefficient: 2, maximumInterval: "3600 seconds", }, }); From 357545f406a3b5b8e9910e11b0592b2612b91193 Mon Sep 17 00:00:00 2001 From: Aubin <60398825+aubin-tchoi@users.noreply.github.com> Date: Thu, 2 Jan 2025 08:08:46 +0100 Subject: [PATCH 07/23] [KW-search] Enforce always passing parentId in connectors (#9580) * add parentId wherever missing * make parentId mandatory in upsertDataSourceDocument and upsertDataSourceFolder * pass parentId when upserting tables * make parentId mandatory in upsertDataSourceTableFromCsv * pass parentId in updateDataSourceDocumentParents and updateDataSourceTableParents * make parentId mandatory updateDataSourceDocumentParents and updateDataSourceTableParents * add parentId in the migration scripts to please the linter * rename a variable for consistency --- .../20230906_2_slack_fill_parents_field.ts | 1 + .../20230906_3_github_fill_parents_field.ts | 2 + ...2_github_add_issues_discussions_parents.ts | 2 + .../migrations/20240802_table_parents.ts | 1 + ...20240828_microsoft_refill_parents_field.ts | 1 + .../migrations/20241030_fix_notion_parents.ts | 2 + .../migrations/20241211_fix_gdrive_parents.ts | 2 + .../20241212_restore_gdrive_parents.ts | 1 + .../20241216_backfill_confluence_folders.ts | 1 + .../20241216_backfill_zendesk_folders.ts | 4 ++ .../20241218_backfill_webcrawler_folders.ts | 4 +- .../20241219_backfill_github_folders.ts | 5 ++ ...9_backfill_intercom_data_source_folders.ts | 4 ++ .../connectors/confluence/lib/hierarchy.ts | 2 +- .../confluence/temporal/activities.ts | 3 +- .../connectors/github/temporal/activities.ts | 53 ++++++++++++------- .../google_drive/temporal/activities.ts | 2 + .../connectors/google_drive/temporal/file.ts | 1 + .../google_drive/temporal/spreadsheets.ts | 1 + .../src/connectors/intercom/lib/utils.ts | 19 ++++--- .../intercom/temporal/activities.ts | 9 ++-- .../intercom/temporal/sync_conversation.ts | 13 +++-- .../intercom/temporal/sync_help_center.ts | 3 +- .../microsoft/temporal/activities.ts | 8 ++- .../src/connectors/microsoft/temporal/file.ts | 3 +- .../microsoft/temporal/spreadsheets.ts | 5 +- .../src/connectors/notion/lib/parents.ts | 1 + .../connectors/notion/temporal/activities.ts | 12 +++-- connectors/src/connectors/shared/file.ts | 4 +- .../webcrawler/temporal/activities.ts | 12 +++-- .../connectors/zendesk/lib/sync_article.ts | 4 +- .../connectors/zendesk/lib/sync_category.ts | 1 + .../src/connectors/zendesk/lib/sync_ticket.ts | 4 +- .../connectors/zendesk/temporal/activities.ts | 4 ++ .../temporal/incremental_activities.ts | 1 + connectors/src/lib/data_sources.ts | 19 +++---- 36 files changed, 146 insertions(+), 68 deletions(-) diff --git a/connectors/migrations/20230906_2_slack_fill_parents_field.ts b/connectors/migrations/20230906_2_slack_fill_parents_field.ts index acff8731768d..2c278bb91fd4 100644 --- a/connectors/migrations/20230906_2_slack_fill_parents_field.ts +++ b/connectors/migrations/20230906_2_slack_fill_parents_field.ts @@ -76,6 +76,7 @@ async function updateParentsFieldForConnector(connector: ConnectorModel) { dataSourceConfig: connector, documentId: documentIdAndChannel.documentId, parents: [documentIdAndChannel.channelId], + parentId: null, }) ) ); diff --git a/connectors/migrations/20230906_3_github_fill_parents_field.ts b/connectors/migrations/20230906_3_github_fill_parents_field.ts index e4042f7ff09e..36d1f35341c2 100644 --- a/connectors/migrations/20230906_3_github_fill_parents_field.ts +++ b/connectors/migrations/20230906_3_github_fill_parents_field.ts @@ -88,6 +88,7 @@ async function updateDiscussionsParentsFieldForConnector( getDiscussionInternalId(document.repoId, document.discussionNumber), document.repoId, ], + parentId: document.repoId, }); }) ); @@ -118,6 +119,7 @@ async function updateIssuesParentsFieldForConnector(connector: ConnectorModel) { getIssueInternalId(document.repoId, document.issueNumber), document.repoId, ], + parentId: document.repoId, }); }) ); diff --git a/connectors/migrations/20240102_github_add_issues_discussions_parents.ts b/connectors/migrations/20240102_github_add_issues_discussions_parents.ts index 776fb6809aac..e1994812781f 100644 --- a/connectors/migrations/20240102_github_add_issues_discussions_parents.ts +++ b/connectors/migrations/20240102_github_add_issues_discussions_parents.ts @@ -48,6 +48,7 @@ async function updateParents(connector: ConnectorModel) { dataSourceConfig: connector, documentId, parents, + parentId: `${d.repoId}-discussions`, }); console.log(`Updated discussion ${documentId} with: ${parents}`); } else { @@ -78,6 +79,7 @@ async function updateParents(connector: ConnectorModel) { dataSourceConfig: connector, documentId, parents, + parentId: `${i.repoId}-issues`, }); console.log(`Updated issue ${documentId} with: ${parents}`); } else { diff --git a/connectors/migrations/20240802_table_parents.ts b/connectors/migrations/20240802_table_parents.ts index f63e1444bb30..f85e0463a99e 100644 --- a/connectors/migrations/20240802_table_parents.ts +++ b/connectors/migrations/20240802_table_parents.ts @@ -49,6 +49,7 @@ async function updateParents({ tableId, parents, dataSourceConfig, + parentId: parents[1] || null, }); } } diff --git a/connectors/migrations/20240828_microsoft_refill_parents_field.ts b/connectors/migrations/20240828_microsoft_refill_parents_field.ts index 8bf907ae2acf..180739cfa3fc 100644 --- a/connectors/migrations/20240828_microsoft_refill_parents_field.ts +++ b/connectors/migrations/20240828_microsoft_refill_parents_field.ts @@ -80,6 +80,7 @@ async function updateParentsFieldForConnector( dataSourceConfig: dataSourceConfigFromConnector(connector), documentId: node.internalId, parents, + parentId: parents[1] || null, }); }, { concurrency: 8 } diff --git a/connectors/migrations/20241030_fix_notion_parents.ts b/connectors/migrations/20241030_fix_notion_parents.ts index 0a48777cbd3a..f12395eae440 100644 --- a/connectors/migrations/20241030_fix_notion_parents.ts +++ b/connectors/migrations/20241030_fix_notion_parents.ts @@ -155,6 +155,7 @@ async function updateParentsFieldForConnector( dataSourceConfig: dataSourceConfigFromConnector(connector), documentId, parents, + parentId: parents[1] || null, retries: 3, }); } @@ -163,6 +164,7 @@ async function updateParentsFieldForConnector( dataSourceConfig: dataSourceConfigFromConnector(connector), tableId, parents, + parentId: parents[1] || null, retries: 3, }); } diff --git a/connectors/migrations/20241211_fix_gdrive_parents.ts b/connectors/migrations/20241211_fix_gdrive_parents.ts index 7bd864d14280..c7bba36f41fc 100644 --- a/connectors/migrations/20241211_fix_gdrive_parents.ts +++ b/connectors/migrations/20241211_fix_gdrive_parents.ts @@ -148,6 +148,7 @@ async function migrate({ dataSourceConfig, tableId, parents, + parentId: parents[1] || null, }); } } @@ -210,6 +211,7 @@ async function migrate({ dataSourceConfig, tableId, parents, + parentId: parents[1] || null, }); } } diff --git a/connectors/migrations/20241212_restore_gdrive_parents.ts b/connectors/migrations/20241212_restore_gdrive_parents.ts index 4a0a1930b7c2..06a7adf652d7 100644 --- a/connectors/migrations/20241212_restore_gdrive_parents.ts +++ b/connectors/migrations/20241212_restore_gdrive_parents.ts @@ -44,6 +44,7 @@ async function processLogFile( dataSourceConfig, documentId: documentId, parents: previousParents, + parentId: previousParents[1] || null, }); } } diff --git a/connectors/migrations/20241216_backfill_confluence_folders.ts b/connectors/migrations/20241216_backfill_confluence_folders.ts index 17a07ca3c716..9e739a6d0d5b 100644 --- a/connectors/migrations/20241216_backfill_confluence_folders.ts +++ b/connectors/migrations/20241216_backfill_confluence_folders.ts @@ -26,6 +26,7 @@ makeScript({}, async ({ execute }, logger) => { dataSourceConfig, folderId: makeSpaceInternalId(space.spaceId), parents: [makeSpaceInternalId(space.spaceId)], + parentId: null, title: space.name, mimeType: "application/vnd.dust.confluence.space", }); diff --git a/connectors/migrations/20241216_backfill_zendesk_folders.ts b/connectors/migrations/20241216_backfill_zendesk_folders.ts index 71359fbb25e4..97418be6ea24 100644 --- a/connectors/migrations/20241216_backfill_zendesk_folders.ts +++ b/connectors/migrations/20241216_backfill_zendesk_folders.ts @@ -33,6 +33,7 @@ makeScript({}, async ({ execute }, logger) => { dataSourceConfig, folderId: brandInternalId, parents: [brandInternalId], + parentId: null, title: brand.name, mimeType: "application/vnd.dust.zendesk.brand", }); @@ -45,6 +46,7 @@ makeScript({}, async ({ execute }, logger) => { helpCenterNode.internalId, helpCenterNode.parentInternalId, ], + parentId: helpCenterNode.parentInternalId, title: helpCenterNode.title, mimeType: "application/vnd.dust.zendesk.helpcenter", }); @@ -54,6 +56,7 @@ makeScript({}, async ({ execute }, logger) => { dataSourceConfig, folderId: ticketsNode.internalId, parents: [ticketsNode.internalId, ticketsNode.parentInternalId], + parentId: ticketsNode.parentInternalId, title: ticketsNode.title, mimeType: "application/vnd.dust.zendesk.tickets", }); @@ -81,6 +84,7 @@ makeScript({}, async ({ execute }, logger) => { dataSourceConfig: dataSourceConfigFromConnector(connector), folderId: parents[0], parents, + parentId: parents[1], title: category.name, mimeType: "application/vnd.dust.zendesk.category", }); diff --git a/connectors/migrations/20241218_backfill_webcrawler_folders.ts b/connectors/migrations/20241218_backfill_webcrawler_folders.ts index 0932a3613e2d..ded602b9ab49 100644 --- a/connectors/migrations/20241218_backfill_webcrawler_folders.ts +++ b/connectors/migrations/20241218_backfill_webcrawler_folders.ts @@ -76,11 +76,13 @@ makeScript( execute, }); if (execute) { + const parents = getParents(folder); const result = await upsertDataSourceFolder({ dataSourceConfig, folderId: folder.internalId, timestampMs: folder.updatedAt.getTime(), - parents: getParents(folder), + parents, + parentId: parents[1] || null, title: folder.url, mimeType: "application/vnd.dust.webcrawler.folder", }); diff --git a/connectors/migrations/20241219_backfill_github_folders.ts b/connectors/migrations/20241219_backfill_github_folders.ts index a8ee24170d11..f6d3ab58bc23 100644 --- a/connectors/migrations/20241219_backfill_github_folders.ts +++ b/connectors/migrations/20241219_backfill_github_folders.ts @@ -42,6 +42,7 @@ async function upsertFoldersForConnector( dataSourceConfig, folderId: repoInternalId, parents: [repoInternalId], + parentId: null, title: repoName, mimeType: "application/vnd.dust.github.repository", }); @@ -61,6 +62,7 @@ async function upsertFoldersForConnector( dataSourceConfig, folderId: issuesInternalId, parents: [issuesInternalId, repoInternalId], + parentId: repoInternalId, title: "Issues", mimeType: "application/vnd.dust.github.issues", }); @@ -76,6 +78,7 @@ async function upsertFoldersForConnector( dataSourceConfig, folderId: discussionsInternalId, parents: [discussionsInternalId, repoInternalId], + parentId: repoInternalId, title: "Discussions", mimeType: "application/vnd.dust.github.discussions", }); @@ -96,6 +99,7 @@ async function upsertFoldersForConnector( folderId: codeRootInternalId, title: "Code", parents: [codeRootInternalId, repoInternalId], + parentId: repoInternalId, mimeType: "application/vnd.dust.github.code.root", }); logger.info(`Upserted code root folder ${codeRootInternalId}`); @@ -121,6 +125,7 @@ async function upsertFoldersForConnector( dataSourceConfig, folderId: directory.internalId, parents: [directory.internalId, ...dirParents], + parentId: dirParents[0] || null, title: directory.dirName, mimeType: "application/vnd.dust.github.code.directory", }); diff --git a/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts b/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts index c54c3e088da0..d0ed53d635dc 100644 --- a/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts +++ b/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts @@ -34,6 +34,7 @@ async function createFolderNodes(execute: boolean) { dataSourceConfig, folderId: getTeamsInternalId(connector.id), parents: [getTeamsInternalId(connector.id)], + parentId: null, title: "Conversations", mimeType: getDataSourceNodeMimeType("CONVERSATIONS_FOLDER"), }); @@ -57,6 +58,7 @@ async function createFolderNodes(execute: boolean) { dataSourceConfig, folderId: teamInternalId, parents: [teamInternalId, getTeamsInternalId(connector.id)], + parentId: getTeamsInternalId(connector.id), title: team.name, mimeType: getDataSourceNodeMimeType("TEAM"), }); @@ -95,6 +97,7 @@ async function createFolderNodes(execute: boolean) { dataSourceConfig, folderId: helpCenterInternalId, parents: [helpCenterInternalId], + parentId: null, title: helpCenter.name, mimeType: getDataSourceNodeMimeType("HELP_CENTER"), }); @@ -128,6 +131,7 @@ async function createFolderNodes(execute: boolean) { dataSourceConfig, folderId: collectionInternalId, parents: collectionParents, + parentId: collectionParents[1] || null, title: collection.name, mimeType: getDataSourceNodeMimeType("COLLECTION"), }); diff --git a/connectors/src/connectors/confluence/lib/hierarchy.ts b/connectors/src/connectors/confluence/lib/hierarchy.ts index 53912b0d42aa..d0c986c4f52b 100644 --- a/connectors/src/connectors/confluence/lib/hierarchy.ts +++ b/connectors/src/connectors/confluence/lib/hierarchy.ts @@ -41,7 +41,7 @@ export async function getConfluencePageParentIds( connectorId: ModelId, page: RawConfluencePage, cachedHierarchy?: Record -) { +): Promise<[string, ...string[], string]> { const pageIdToParentIdMap = cachedHierarchy ?? (await getSpaceHierarchy(connectorId, page.spaceId)); diff --git a/connectors/src/connectors/confluence/temporal/activities.ts b/connectors/src/connectors/confluence/temporal/activities.ts index 5819132abc4d..dfbcb88955f2 100644 --- a/connectors/src/connectors/confluence/temporal/activities.ts +++ b/connectors/src/connectors/confluence/temporal/activities.ts @@ -219,6 +219,7 @@ export async function confluenceUpsertSpaceFolderActivity({ dataSourceConfig: dataSourceConfigFromConnector(connector), folderId: makeSpaceInternalId(spaceId), parents: [makeSpaceInternalId(spaceId)], + parentId: null, title: spaceName, mimeType: "application/vnd.dust.confluence.space", }); @@ -252,7 +253,7 @@ export async function markPageHasVisited({ interface ConfluenceUpsertPageInput { page: NonNullable>>; spaceName: string; - parents: string[]; + parents: [string, ...string[], string]; confluenceConfig: ConfluenceConfiguration; syncType?: UpsertDataSourceDocumentParams["upsertContext"]["sync_type"]; dataSourceConfig: DataSourceConfig; diff --git a/connectors/src/connectors/github/temporal/activities.ts b/connectors/src/connectors/github/temporal/activities.ts index 1e297105fe21..487cb08a7eb8 100644 --- a/connectors/src/connectors/github/temporal/activities.ts +++ b/connectors/src/connectors/github/temporal/activities.ts @@ -288,6 +288,11 @@ export async function githubUpsertIssueActivity( tags.push(`author:${issueAuthor}`); } + const parents: [string, string, string] = [ + documentId, + getIssuesInternalId(repoId), + getRepositoryInternalId(repoId), + ]; // TODO: last commentor, last comment date, issue labels (as tags) await upsertDataSourceDocument({ dataSourceConfig, @@ -296,11 +301,8 @@ export async function githubUpsertIssueActivity( documentUrl: issue.url, timestampMs: updatedAtTimestamp, tags: tags, - parents: [ - documentId, - getIssuesInternalId(repoId), - getRepositoryInternalId(repoId), - ], + parents, + parentId: parents[1], loggerArgs: logger.bindings(), upsertContext: { sync_type: isBatchSync ? "batch" : "incremental", @@ -473,6 +475,11 @@ export async function githubUpsertDiscussionActivity( `updatedAt:${new Date(discussion.updatedAt).getTime()}`, ]; + const parents: [string, string, string] = [ + documentId, + getDiscussionsInternalId(repoId), + getRepositoryInternalId(repoId), + ]; await upsertDataSourceDocument({ dataSourceConfig, documentId, @@ -480,11 +487,8 @@ export async function githubUpsertDiscussionActivity( documentUrl: discussion.url, timestampMs: new Date(discussion.createdAt).getTime(), tags, - parents: [ - documentId, - getDiscussionsInternalId(repoId), - getRepositoryInternalId(repoId), - ], + parents, + parentId: parents[1], loggerArgs: logger.bindings(), upsertContext: { sync_type: isBatchSync ? "batch" : "incremental", @@ -955,6 +959,7 @@ export async function githubCodeSyncActivity({ folderId: getCodeRootInternalId(repoId), title: "Code", parents: [getCodeRootInternalId(repoId), getRepositoryInternalId(repoId)], + parentId: getRepositoryInternalId(repoId), mimeType: "application/vnd.dust.github.code.root", }); @@ -1147,6 +1152,11 @@ export async function githubCodeSyncActivity({ `lasUpdatedAt:${codeSyncStartedAt.getTime()}`, ]; + const parents: [...string[], string, string] = [ + ...f.parents, + rootInternalId, + getRepositoryInternalId(repoId), + ]; // Time to upload the file to the data source. await upsertDataSourceDocument({ dataSourceConfig, @@ -1155,11 +1165,8 @@ export async function githubCodeSyncActivity({ documentUrl: f.sourceUrl, timestampMs: codeSyncStartedAt.getTime(), tags, - parents: [ - ...f.parents, - rootInternalId, - getRepositoryInternalId(repoId), - ], + parents, + parentId: parents[1], loggerArgs: logger.bindings(), upsertContext: { sync_type: isBatchSync ? "batch" : "incremental", @@ -1198,14 +1205,16 @@ export async function githubCodeSyncActivity({ Context.current().heartbeat(); const parentInternalId = d.parentInternalId || rootInternalId; + const parents: [...string[], string, string] = [ + ...d.parents, + getCodeRootInternalId(repoId), + getRepositoryInternalId(repoId), + ]; await upsertDataSourceFolder({ dataSourceConfig, folderId: d.internalId, - parents: [ - ...d.parents, - getCodeRootInternalId(repoId), - getRepositoryInternalId(repoId), - ], + parents, + parentId: parents[1], title: d.dirName, mimeType: "application/vnd.dust.github.code.directory", }); @@ -1346,6 +1355,7 @@ export async function githubUpsertRepositoryFolderActivity({ folderId: getRepositoryInternalId(repoId), title: repoName, parents: [getRepositoryInternalId(repoId)], + parentId: null, mimeType: "application/vnd.dust.github.repository", }); } @@ -1366,6 +1376,7 @@ export async function githubUpsertIssuesFolderActivity({ folderId: getIssuesInternalId(repoId), title: "Issues", parents: [getIssuesInternalId(repoId), getRepositoryInternalId(repoId)], + parentId: getRepositoryInternalId(repoId), mimeType: "application/vnd.dust.github.issues", }); } @@ -1389,6 +1400,7 @@ export async function githubUpsertDiscussionsFolderActivity({ getDiscussionsInternalId(repoId), getRepositoryInternalId(repoId), ], + parentId: getRepositoryInternalId(repoId), mimeType: "application/vnd.dust.github.discussions", }); } @@ -1409,6 +1421,7 @@ export async function githubUpsertCodeRootFolderActivity({ folderId: getCodeRootInternalId(repoId), title: "Code", parents: [getCodeRootInternalId(repoId), getRepositoryInternalId(repoId)], + parentId: getRepositoryInternalId(repoId), mimeType: "application/vnd.dust.github.code.root", }); } diff --git a/connectors/src/connectors/google_drive/temporal/activities.ts b/connectors/src/connectors/google_drive/temporal/activities.ts index 45566089dc52..65e1a2859835 100644 --- a/connectors/src/connectors/google_drive/temporal/activities.ts +++ b/connectors/src/connectors/google_drive/temporal/activities.ts @@ -512,6 +512,7 @@ export async function incrementalSync( dataSourceConfig, folderId: getInternalId(driveFile.id), parents, + parentId: parents[1] || null, title: driveFile.name ?? "", mimeType: "application/vnd.dust.googledrive.folder", }); @@ -856,6 +857,7 @@ export async function markFolderAsVisited( dataSourceConfig, folderId: getInternalId(file.id), parents, + parentId: parents[1] || null, title: file.name ?? "", mimeType: "application/vnd.dust.googledrive.folder", }); diff --git a/connectors/src/connectors/google_drive/temporal/file.ts b/connectors/src/connectors/google_drive/temporal/file.ts index eb327593e00f..2f5d918b4fcc 100644 --- a/connectors/src/connectors/google_drive/temporal/file.ts +++ b/connectors/src/connectors/google_drive/temporal/file.ts @@ -492,6 +492,7 @@ async function upsertGdriveDocument( timestampMs: file.updatedAtMs, tags, parents, + parentId: parents[1] || null, upsertContext: { sync_type: isBatchSync ? "batch" : "incremental", }, diff --git a/connectors/src/connectors/google_drive/temporal/spreadsheets.ts b/connectors/src/connectors/google_drive/temporal/spreadsheets.ts index d55815aab711..7b1e89cbcef9 100644 --- a/connectors/src/connectors/google_drive/temporal/spreadsheets.ts +++ b/connectors/src/connectors/google_drive/temporal/spreadsheets.ts @@ -83,6 +83,7 @@ async function upsertGdriveTable( }, truncate: true, parents: [tableId, ...parents], + parentId: parents[0] || null, useAppForHeaderDetection: true, title: `${spreadsheet.title} - ${title}`, mimeType: "application/vnd.google-apps.spreadsheet", diff --git a/connectors/src/connectors/intercom/lib/utils.ts b/connectors/src/connectors/intercom/lib/utils.ts index 2db0c96a85ad..a763145be6e5 100644 --- a/connectors/src/connectors/intercom/lib/utils.ts +++ b/connectors/src/connectors/intercom/lib/utils.ts @@ -149,7 +149,7 @@ export async function getParentIdsForArticle({ connectorId: number; parentCollectionId: string; helpCenterId: string; -}) { +}): Promise<[string, string, ...string[], string]> { // Get collection parents const collectionParents = await getParentIdsForCollection({ connectorId, @@ -168,11 +168,8 @@ export async function getParentIdsForCollection({ connectorId: number; collectionId: string; helpCenterId: string; -}) { - // Initialize the internal IDs array with the collection ID. - const parentIds = [ - getHelpCenterCollectionInternalId(connectorId, collectionId), - ]; +}): Promise<[string, ...string[], string]> { + const parentIds = []; // Fetch and add any parent collection Ids. let currentParentId = collectionId; @@ -196,8 +193,10 @@ export async function getParentIdsForCollection({ ); } - // Add the help center internal ID. - parentIds.push(getHelpCenterInternalId(connectorId, helpCenterId)); - - return parentIds; + // Add the collection ID and the help center internal ID. + return [ + getHelpCenterCollectionInternalId(connectorId, collectionId), + ...parentIds, + getHelpCenterInternalId(connectorId, helpCenterId), + ]; } diff --git a/connectors/src/connectors/intercom/temporal/activities.ts b/connectors/src/connectors/intercom/temporal/activities.ts index c1b27952385b..de1b06a85ed0 100644 --- a/connectors/src/connectors/intercom/temporal/activities.ts +++ b/connectors/src/connectors/intercom/temporal/activities.ts @@ -30,14 +30,12 @@ import { import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config"; import { concurrentExecutor } from "@connectors/lib/async_utils"; import { upsertDataSourceFolder } from "@connectors/lib/data_sources"; -import { - IntercomConversation, - IntercomWorkspace, -} from "@connectors/lib/models/intercom"; import { IntercomCollection, + IntercomConversation, IntercomHelpCenter, IntercomTeam, + IntercomWorkspace, } from "@connectors/lib/models/intercom"; import { syncStarted, syncSucceeded } from "@connectors/lib/sync_status"; import logger from "@connectors/logger/logger"; @@ -177,6 +175,7 @@ export async function syncHelpCenterOnlyActivity({ folderId: helpCenterInternalId, title: helpCenterOnIntercom.display_name || "Help Center", parents: [helpCenterInternalId], + parentId: null, mimeType: getDataSourceNodeMimeType("HELP_CENTER"), }); @@ -509,6 +508,7 @@ export async function syncTeamOnlyActivity({ folderId: teamInternalId, title: teamOnIntercom.name, parents: [teamInternalId, getTeamsInternalId(connectorId)], + parentId: getTeamsInternalId(connectorId), mimeType: getDataSourceNodeMimeType("TEAM"), }); @@ -743,6 +743,7 @@ export async function upsertIntercomTeamsFolderActivity({ folderId: getTeamsInternalId(connectorId), title: "Conversations", parents: [getTeamsInternalId(connectorId)], + parentId: null, mimeType: getDataSourceNodeMimeType("CONVERSATIONS_FOLDER"), }); } diff --git a/connectors/src/connectors/intercom/temporal/sync_conversation.ts b/connectors/src/connectors/intercom/temporal/sync_conversation.ts index b68b03f9b6ac..56b00b70447a 100644 --- a/connectors/src/connectors/intercom/temporal/sync_conversation.ts +++ b/connectors/src/connectors/intercom/temporal/sync_conversation.ts @@ -306,11 +306,13 @@ export async function syncConversation({ // parents in the Core datasource map the internal ids that are used in the permission system // they self reference the document id const documentId = getConversationInternalId(connectorId, conversation.id); - const parents = [documentId]; - if (conversationTeamId) { - parents.push(getTeamInternalId(connectorId, conversationTeamId)); - } - parents.push(getTeamsInternalId(connectorId)); + const parents: [string, ...string[], string] = [ + documentId, + ...(conversationTeamId + ? [getTeamInternalId(connectorId, conversationTeamId)] + : []), + getTeamsInternalId(connectorId), + ]; await upsertDataSourceDocument({ dataSourceConfig, @@ -320,6 +322,7 @@ export async function syncConversation({ timestampMs: updatedAtDate.getTime(), tags: datasourceTags, parents, + parentId: parents[1], loggerArgs: { ...loggerArgs, conversationId: conversation.id, diff --git a/connectors/src/connectors/intercom/temporal/sync_help_center.ts b/connectors/src/connectors/intercom/temporal/sync_help_center.ts index db50d4e0ca49..8aa9d21dd94c 100644 --- a/connectors/src/connectors/intercom/temporal/sync_help_center.ts +++ b/connectors/src/connectors/intercom/temporal/sync_help_center.ts @@ -228,7 +228,7 @@ export async function upsertCollectionWithChildren({ folderId: internalCollectionId, title: collection.name, parents: collectionParents, - parentId: collectionParents.length > 2 ? collectionParents[1] : null, + parentId: collectionParents[1], mimeType: getDataSourceNodeMimeType("COLLECTION"), }); @@ -420,6 +420,7 @@ export async function upsertArticle({ `updatedAt:${updatedAtDate.getTime()}`, ], parents, + parentId: parents[1], loggerArgs: { ...loggerArgs, articleId: article.id, diff --git a/connectors/src/connectors/microsoft/temporal/activities.ts b/connectors/src/connectors/microsoft/temporal/activities.ts index 2557cae78110..88dc0bc727ce 100644 --- a/connectors/src/connectors/microsoft/temporal/activities.ts +++ b/connectors/src/connectors/microsoft/temporal/activities.ts @@ -207,6 +207,7 @@ export async function getRootNodesToSyncFromResources( dataSourceConfig, folderId: createdOrUpdatedResource.internalId, parents: [createdOrUpdatedResource.internalId], + parentId: null, title: createdOrUpdatedResource.name ?? "", mimeType: "application/vnd.dust.microsoft.folder", }), @@ -465,7 +466,7 @@ export async function syncFiles({ ) ); - const parents = await getParents({ + const parentsOfParent = await getParents({ connectorId: parent.connectorId, internalId: parent.internalId, startSyncTs, @@ -477,7 +478,8 @@ export async function syncFiles({ upsertDataSourceFolder({ dataSourceConfig, folderId: createdOrUpdatedResource.internalId, - parents: [createdOrUpdatedResource.internalId, ...parents], + parents: [createdOrUpdatedResource.internalId, ...parentsOfParent], + parentId: parentsOfParent[0], title: createdOrUpdatedResource.name ?? "", mimeType: "application/vnd.dust.microsoft.folder", }), @@ -651,6 +653,7 @@ export async function syncDeltaForRootNodesInDrive({ dataSourceConfig, folderId: blob.internalId, parents: [blob.internalId], + parentId: null, title: blob.name ?? "", mimeType: "application/vnd.dust.microsoft.folder", }); @@ -904,6 +907,7 @@ async function updateParentsField({ dataSourceConfig, documentId: file.internalId, parents, + parentId: parents[1] || null, }); } diff --git a/connectors/src/connectors/microsoft/temporal/file.ts b/connectors/src/connectors/microsoft/temporal/file.ts index 1eb3ee24636c..2543c013f8d8 100644 --- a/connectors/src/connectors/microsoft/temporal/file.ts +++ b/connectors/src/connectors/microsoft/temporal/file.ts @@ -306,6 +306,7 @@ export async function syncOneFile({ timestampMs: upsertTimestampMs, tags, parents, + parentId: parents[1] || null, upsertContext: { sync_type: isBatchSync ? "batch" : "incremental", }, @@ -352,7 +353,7 @@ export async function getParents({ connectorId: ModelId; internalId: string; startSyncTs: number; -}): Promise { +}): Promise<[string, ...string[]]> { const parentInternalId = await getParentId( connectorId, internalId, diff --git a/connectors/src/connectors/microsoft/temporal/spreadsheets.ts b/connectors/src/connectors/microsoft/temporal/spreadsheets.ts index 6ae10afa9cf5..303926dfa250 100644 --- a/connectors/src/connectors/microsoft/temporal/spreadsheets.ts +++ b/connectors/src/connectors/microsoft/temporal/spreadsheets.ts @@ -70,7 +70,7 @@ async function upsertMSTable( internalId: string, spreadsheet: microsoftgraph.DriveItem, worksheet: microsoftgraph.WorkbookWorksheet, - parents: string[], + parents: [string, string, ...string[]], rows: string[][], loggerArgs: object ) { @@ -99,6 +99,7 @@ async function upsertMSTable( }, truncate: true, parents, + parentId: parents[1], useAppForHeaderDetection: true, title: `${spreadsheet.name} - ${worksheet.name}`, mimeType: @@ -183,7 +184,7 @@ async function processSheet({ // Assuming the first line as headers, at least one additional data line is required. if (rows.length > 1) { - const parents = [ + const parents: [string, string, ...string[]] = [ worksheetInternalId, ...(await getParents({ connectorId: connector.id, diff --git a/connectors/src/connectors/notion/lib/parents.ts b/connectors/src/connectors/notion/lib/parents.ts index 04574447789a..6ea2ee47bf92 100644 --- a/connectors/src/connectors/notion/lib/parents.ts +++ b/connectors/src/connectors/notion/lib/parents.ts @@ -176,6 +176,7 @@ export async function updateAllParentsFields( dataSourceConfig: dataSourceConfigFromConnector(connector), documentId: `notion-${pageId}`, parents, + parentId: parents[1] || null, }); if (onProgress) { await onProgress(); diff --git a/connectors/src/connectors/notion/temporal/activities.ts b/connectors/src/connectors/notion/temporal/activities.ts index 12ab757f4ed9..77c0a1714e79 100644 --- a/connectors/src/connectors/notion/temporal/activities.ts +++ b/connectors/src/connectors/notion/temporal/activities.ts @@ -1,5 +1,9 @@ -import type { CoreAPIDataSourceDocumentSection, ModelId } from "@dust-tt/types"; -import type { PageObjectProperties, ParsedNotionBlock } from "@dust-tt/types"; +import type { + CoreAPIDataSourceDocumentSection, + ModelId, + PageObjectProperties, + ParsedNotionBlock, +} from "@dust-tt/types"; import { assertNever, getNotionDatabaseTableId, slugify } from "@dust-tt/types"; import { isFullBlock, isFullPage, isNotionClientError } from "@notionhq/client"; import type { PageObjectResponse } from "@notionhq/client/build/src/api-endpoints"; @@ -1818,6 +1822,7 @@ export async function renderAndUpsertPageFromCache({ // We only update the rowId of for the page without truncating the rest of the table (incremental sync). truncate: false, parents: parents, + parentId: parents[1] || null, title: parentDb.title ?? "Untitled Notion Database", mimeType: "application/vnd.dust.notion.database", }), @@ -2037,7 +2042,7 @@ export async function renderAndUpsertPageFromCache({ parsedProperties, }), parents: parentIds, - parentId: parentIds.length > 1 ? parentIds[1] : null, + parentId: parentIds[1] || null, loggerArgs, upsertContext: { sync_type: isFullSync ? "batch" : "incremental", @@ -2538,6 +2543,7 @@ export async function upsertDatabaseStructuredDataFromCache({ // We overwrite the whole table since we just fetched all child pages. truncate: true, parents: parentIds, + parentId: parentIds[1] || null, title: dbModel.title ?? "Untitled Notion Database", mimeType: "application/vnd.dust.notion.database", }), diff --git a/connectors/src/connectors/shared/file.ts b/connectors/src/connectors/shared/file.ts index a84edffcca7b..d845cce13a13 100644 --- a/connectors/src/connectors/shared/file.ts +++ b/connectors/src/connectors/shared/file.ts @@ -10,9 +10,10 @@ import { isTextExtractionSupportedContentType, Ok, pagePrefixesPerMimeType, + parseAndStringifyCsv, + slugify, TextExtraction, } from "@dust-tt/types"; -import { parseAndStringifyCsv, slugify } from "@dust-tt/types"; import { apiConfig } from "@connectors/lib/api/config"; import { upsertDataSourceTableFromCsv } from "@connectors/lib/data_sources"; @@ -78,6 +79,7 @@ export async function handleCsvFile({ }, truncate: true, parents, + parentId: parents[1] || null, title: fileName, mimeType: "text/csv", }); diff --git a/connectors/src/connectors/webcrawler/temporal/activities.ts b/connectors/src/connectors/webcrawler/temporal/activities.ts index cea57969e776..b27200375a3d 100644 --- a/connectors/src/connectors/webcrawler/temporal/activities.ts +++ b/connectors/src/connectors/webcrawler/temporal/activities.ts @@ -279,15 +279,16 @@ export async function crawlWebsiteByConnectorId(connectorId: ModelId) { lastSeenAt: new Date(), }); + // parent folder ids of the page are in hierarchy order from the + // page to the root so for the current folder, its parents start at + // index+1 (including itself as first parent) and end at the root + const parents = parentFolderIds.slice(index + 1); await upsertDataSourceFolder({ dataSourceConfig, folderId: webCrawlerFolder.internalId, timestampMs: webCrawlerFolder.updatedAt.getTime(), - - // parent folder ids of the page are in hierarchy order from the - // page to the root so for the current folder, its parents start at - // index+1 (including itself as first parent) and end at the root - parents: parentFolderIds.slice(index + 1), + parents, + parentId: parents[1] || null, title: folder, mimeType: "application/vnd.dust.webcrawler.folder", }); @@ -363,6 +364,7 @@ export async function crawlWebsiteByConnectorId(connectorId: ModelId) { timestampMs: new Date().getTime(), tags: [`title:${stripNullBytes(pageTitle)}`], parents: parentFolderIds, + parentId: parentFolderIds[1] || null, upsertContext: { sync_type: "batch", }, diff --git a/connectors/src/connectors/zendesk/lib/sync_article.ts b/connectors/src/connectors/zendesk/lib/sync_article.ts index 79770c7cd7ee..ed3d7ddabbf6 100644 --- a/connectors/src/connectors/zendesk/lib/sync_article.ts +++ b/connectors/src/connectors/zendesk/lib/sync_article.ts @@ -151,6 +151,7 @@ export async function syncArticle({ articleId: article.id, }); + const parents = articleInDb.getParentInternalIds(connectorId); await upsertDataSourceDocument({ dataSourceConfig, documentId, @@ -162,7 +163,8 @@ export async function syncArticle({ `createdAt:${createdAt.getTime()}`, `updatedAt:${updatedAt.getTime()}`, ], - parents: articleInDb.getParentInternalIds(connectorId), + parents, + parentId: parents[1], loggerArgs: { ...loggerArgs, articleId: article.id }, upsertContext: { sync_type: "batch" }, title: article.title, diff --git a/connectors/src/connectors/zendesk/lib/sync_category.ts b/connectors/src/connectors/zendesk/lib/sync_category.ts index ed27450f20d6..55f9b23e082b 100644 --- a/connectors/src/connectors/zendesk/lib/sync_category.ts +++ b/connectors/src/connectors/zendesk/lib/sync_category.ts @@ -104,6 +104,7 @@ export async function syncCategory({ dataSourceConfig, folderId: parents[0], parents, + parentId: parents[1], title: categoryInDb.name, mimeType: "application/vnd.dust.zendesk.category", }); diff --git a/connectors/src/connectors/zendesk/lib/sync_ticket.ts b/connectors/src/connectors/zendesk/lib/sync_ticket.ts index 2d5dc17d33c6..b626b53fd5c6 100644 --- a/connectors/src/connectors/zendesk/lib/sync_ticket.ts +++ b/connectors/src/connectors/zendesk/lib/sync_ticket.ts @@ -209,6 +209,7 @@ ${comments ticketId: ticket.id, }); + const parents = ticketInDb.getParentInternalIds(connectorId); await upsertDataSourceDocument({ dataSourceConfig, documentId, @@ -221,7 +222,8 @@ ${comments `updatedAt:${updatedAtDate.getTime()}`, `createdAt:${createdAtDate.getTime()}`, ], - parents: ticketInDb.getParentInternalIds(connectorId), + parents, + parentId: parents[1], loggerArgs: { ...loggerArgs, ticketId: ticket.id }, upsertContext: { sync_type: "batch" }, title: ticket.subject, diff --git a/connectors/src/connectors/zendesk/temporal/activities.ts b/connectors/src/connectors/zendesk/temporal/activities.ts index 1481b3cde5a0..2df641d54c5d 100644 --- a/connectors/src/connectors/zendesk/temporal/activities.ts +++ b/connectors/src/connectors/zendesk/temporal/activities.ts @@ -132,6 +132,7 @@ export async function syncZendeskBrandActivity({ dataSourceConfig, folderId: brandInternalId, parents: [brandInternalId], + parentId: null, title: brandInDb.name, mimeType: "application/vnd.dust.zendesk.brand", }); @@ -142,6 +143,7 @@ export async function syncZendeskBrandActivity({ dataSourceConfig, folderId: helpCenterNode.internalId, parents: [helpCenterNode.internalId, helpCenterNode.parentInternalId], + parentId: helpCenterNode.parentInternalId, title: helpCenterNode.title, mimeType: "application/vnd.dust.zendesk.helpcenter", }); @@ -152,6 +154,7 @@ export async function syncZendeskBrandActivity({ dataSourceConfig, folderId: ticketsNode.internalId, parents: [ticketsNode.internalId, ticketsNode.parentInternalId], + parentId: ticketsNode.parentInternalId, title: ticketsNode.title, mimeType: "application/vnd.dust.zendesk.tickets", }); @@ -326,6 +329,7 @@ export async function syncZendeskCategoryActivity({ dataSourceConfig: dataSourceConfigFromConnector(connector), folderId: parents[0], parents, + parentId: parents[1], title: categoryInDb.name, mimeType: "application/vnd.dust.zendesk.category", }); diff --git a/connectors/src/connectors/zendesk/temporal/incremental_activities.ts b/connectors/src/connectors/zendesk/temporal/incremental_activities.ts index 08fd18c484d9..2522aa32c3c6 100644 --- a/connectors/src/connectors/zendesk/temporal/incremental_activities.ts +++ b/connectors/src/connectors/zendesk/temporal/incremental_activities.ts @@ -147,6 +147,7 @@ export async function syncZendeskArticleUpdateBatchActivity({ dataSourceConfig, folderId: parents[0], parents, + parentId: parents[1], title: category.name, mimeType: "application/vnd.dust.zendesk.category", }); diff --git a/connectors/src/lib/data_sources.ts b/connectors/src/lib/data_sources.ts index d2a2a278a804..43fd720d5633 100644 --- a/connectors/src/lib/data_sources.ts +++ b/connectors/src/lib/data_sources.ts @@ -71,7 +71,7 @@ export type UpsertDataSourceDocumentParams = { timestampMs?: number; tags?: string[]; parents: string[]; - parentId?: string | null; + parentId: string | null; loggerArgs?: Record; upsertContext: UpsertContext; title: string; @@ -108,7 +108,7 @@ async function _upsertDataSourceDocument({ title, mimeType, async, - parentId = null, + parentId, }: UpsertDataSourceDocumentParams) { return tracer.trace( `connectors`, @@ -331,7 +331,7 @@ async function _updateDataSourceDocumentParents({ dataSourceConfig: DataSourceConfig; documentId: string; parents: string[]; - parentId?: string | null; + parentId: string | null; loggerArgs?: Record; }) { return _updateDocumentOrTableParentsField({ @@ -352,6 +352,7 @@ async function _updateDataSourceTableParents({ dataSourceConfig: DataSourceConfig; tableId: string; parents: string[]; + parentId: string | null; loggerArgs?: Record; }) { return _updateDocumentOrTableParentsField({ @@ -365,14 +366,14 @@ async function _updateDocumentOrTableParentsField({ dataSourceConfig, id, parents, - parentId = null, + parentId, loggerArgs = {}, tableOrDocument, }: { dataSourceConfig: DataSourceConfig; id: string; parents: string[]; - parentId?: string | null; + parentId: string | null; loggerArgs?: Record; tableOrDocument: "document" | "table"; }) { @@ -778,7 +779,7 @@ export async function upsertDataSourceTableFromCsv({ loggerArgs, truncate, parents, - parentId = null, + parentId, useAppForHeaderDetection, title, mimeType, @@ -791,7 +792,7 @@ export async function upsertDataSourceTableFromCsv({ loggerArgs?: Record; truncate: boolean; parents: string[]; - parentId?: string | null; + parentId: string | null; useAppForHeaderDetection?: boolean; title: string; mimeType: string; @@ -1229,7 +1230,7 @@ export async function _upsertDataSourceFolder({ folderId, timestampMs, parents, - parentId = parents[1] ?? null, + parentId, title, mimeType, }: { @@ -1237,7 +1238,7 @@ export async function _upsertDataSourceFolder({ folderId: string; timestampMs?: number; parents: string[]; - parentId?: string | null; + parentId: string | null; title: string; mimeType: string; }) { From 6637829a8db991fcf64ada2cff9f1319dfdc4c43 Mon Sep 17 00:00:00 2001 From: Aubin <60398825+aubin-tchoi@users.noreply.github.com> Date: Thu, 2 Jan 2025 08:23:51 +0100 Subject: [PATCH 08/23] add a missing parentId in updateDescendantsParentsInCore (#9683) --- .../src/connectors/microsoft/temporal/activities.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/connectors/src/connectors/microsoft/temporal/activities.ts b/connectors/src/connectors/microsoft/temporal/activities.ts index 88dc0bc727ce..1815014be96b 100644 --- a/connectors/src/connectors/microsoft/temporal/activities.ts +++ b/connectors/src/connectors/microsoft/temporal/activities.ts @@ -860,14 +860,16 @@ async function updateDescendantsParentsInCore({ const files = children.filter((child) => child.nodeType === "file"); const folders = children.filter((child) => child.nodeType === "folder"); + const parents = await getParents({ + connectorId: folder.connectorId, + internalId: folder.internalId, + startSyncTs, + }); await upsertDataSourceFolder({ dataSourceConfig, folderId: folder.internalId, - parents: await getParents({ - connectorId: folder.connectorId, - internalId: folder.internalId, - startSyncTs, - }), + parents, + parentId: parents[1] || null, title: folder.name ?? "", mimeType: "application/vnd.dust.microsoft.folder", }); From 03e91b20497f02a992df4256d51ec65f874f2273 Mon Sep 17 00:00:00 2001 From: Aubin <60398825+aubin-tchoi@users.noreply.github.com> Date: Thu, 2 Jan 2025 08:28:49 +0100 Subject: [PATCH 09/23] enh(Zendesk) - add a CLI command to fetch a single ticket (#9641) * add a CLI command to fetch a single ticket from the Zendesk API * add missing types * remove a layer of types * fix a typo * fix types add ZendeskResyncTicketsResponseSchema to the AdminResponseSchema * fix types add ZendeskFetchTicketResponseSchema to the AdminResponseSchema --- connectors/src/connectors/zendesk/lib/cli.ts | 43 ++++++++++++++++++- .../src/connectors/zendesk/lib/zendesk_api.ts | 24 +++++++++++ types/src/connectors/admin/cli.ts | 12 ++++++ 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/connectors/src/connectors/zendesk/lib/cli.ts b/connectors/src/connectors/zendesk/lib/cli.ts index 82b35853c2b4..6e4c5cc0bc87 100644 --- a/connectors/src/connectors/zendesk/lib/cli.ts +++ b/connectors/src/connectors/zendesk/lib/cli.ts @@ -2,19 +2,24 @@ import type { ZendeskCheckIsAdminResponseType, ZendeskCommandType, ZendeskCountTicketsResponseType, + ZendeskFetchTicketResponseType, ZendeskResyncTicketsResponseType, } from "@dust-tt/types"; import { getZendeskSubdomainAndAccessToken } from "@connectors/connectors/zendesk/lib/zendesk_access_token"; import { fetchZendeskCurrentUser, + fetchZendeskTicket, fetchZendeskTicketCount, getZendeskBrandSubdomain, } from "@connectors/connectors/zendesk/lib/zendesk_api"; import { launchZendeskTicketReSyncWorkflow } from "@connectors/connectors/zendesk/temporal/client"; import { default as topLogger } from "@connectors/logger/logger"; import { ConnectorResource } from "@connectors/resources/connector_resource"; -import { ZendeskConfigurationResource } from "@connectors/resources/zendesk_resources"; +import { + ZendeskConfigurationResource, + ZendeskTicketResource, +} from "@connectors/resources/zendesk_resources"; export const zendesk = async ({ command, @@ -23,6 +28,7 @@ export const zendesk = async ({ | ZendeskCheckIsAdminResponseType | ZendeskCountTicketsResponseType | ZendeskResyncTicketsResponseType + | ZendeskFetchTicketResponseType > => { const logger = topLogger.child({ majorCommand: "zendesk", command, args }); @@ -101,5 +107,40 @@ export const zendesk = async ({ } return { success: true }; } + case "fetch-ticket": { + if (!connector) { + throw new Error(`Connector ${connectorId} not found`); + } + const brandId = args.brandId ? Number(args.brandId) : null; + if (!brandId) { + throw new Error(`Missing --brandId argument`); + } + const ticketId = args.ticketId ? Number(args.ticketId) : null; + if (!ticketId) { + throw new Error(`Missing --ticketId argument`); + } + const { accessToken, subdomain } = + await getZendeskSubdomainAndAccessToken(connector.connectionId); + const brandSubdomain = await getZendeskBrandSubdomain({ + connectorId: connector.id, + brandId, + subdomain, + accessToken, + }); + + const ticket = await fetchZendeskTicket({ + accessToken, + ticketId, + brandSubdomain, + }); + const ticketOnDb = await ZendeskTicketResource.fetchByTicketId({ + connectorId: connector.id, + ticketId, + }); + return { + ticket: ticket as { [key: string]: unknown } | null, + isTicketOnDb: ticketOnDb !== null, + }; + } } }; diff --git a/connectors/src/connectors/zendesk/lib/zendesk_api.ts b/connectors/src/connectors/zendesk/lib/zendesk_api.ts index 8800e5c0c84e..abfd1327c4e7 100644 --- a/connectors/src/connectors/zendesk/lib/zendesk_api.ts +++ b/connectors/src/connectors/zendesk/lib/zendesk_api.ts @@ -349,6 +349,30 @@ export async function fetchZendeskTickets( }; } +/** + * Fetches a single ticket from the Zendesk API. + */ +export async function fetchZendeskTicket({ + accessToken, + brandSubdomain, + ticketId, +}: { + accessToken: string; + brandSubdomain: string; + ticketId: number; +}): Promise { + const url = `https://${brandSubdomain}.zendesk.com/api/v2/tickets/${ticketId}`; + try { + const response = await fetchFromZendeskWithRetries({ url, accessToken }); + return response?.ticket ?? null; + } catch (e) { + if (isZendeskNotFoundError(e)) { + return null; + } + throw e; + } +} + /** * Fetches the number of tickets in a Brand from the Zendesk API. * Only counts tickets that have been solved, and that were updated within the retention period. diff --git a/types/src/connectors/admin/cli.ts b/types/src/connectors/admin/cli.ts index e47ac80db8b0..b394748e37a3 100644 --- a/types/src/connectors/admin/cli.ts +++ b/types/src/connectors/admin/cli.ts @@ -261,12 +261,14 @@ export const ZendeskCommandSchema = t.type({ t.literal("check-is-admin"), t.literal("count-tickets"), t.literal("resync-tickets"), + t.literal("fetch-ticket"), ]), args: t.type({ connectorId: t.union([t.number, t.undefined]), brandId: t.union([t.number, t.undefined]), query: t.union([t.string, t.undefined]), forceResync: t.union([t.literal("true"), t.undefined]), + ticketId: t.union([t.number, t.undefined]), }), }); export type ZendeskCommandType = t.TypeOf; @@ -293,6 +295,14 @@ export const ZendeskResyncTicketsResponseSchema = t.type({ export type ZendeskResyncTicketsResponseType = t.TypeOf< typeof ZendeskResyncTicketsResponseSchema >; + +export const ZendeskFetchTicketResponseSchema = t.type({ + ticket: t.union([t.UnknownRecord, t.null]), // Zendesk type, can't be iots'd, + isTicketOnDb: t.boolean, +}); +export type ZendeskFetchTicketResponseType = t.TypeOf< + typeof ZendeskFetchTicketResponseSchema +>; /** * */ @@ -453,6 +463,8 @@ export const AdminResponseSchema = t.union([ IntercomForceResyncArticlesResponseSchema, ZendeskCheckIsAdminResponseSchema, ZendeskCountTicketsResponseSchema, + ZendeskResyncTicketsResponseSchema, + ZendeskFetchTicketResponseSchema, ]); export type AdminResponseType = t.TypeOf; From 5c7b131a9d3ca5a7bbc2943b9f17de434c63b4b6 Mon Sep 17 00:00:00 2001 From: thib-martin <168569391+thib-martin@users.noreply.github.com> Date: Thu, 2 Jan 2025 09:50:45 +0100 Subject: [PATCH 10/23] fixing button color and trusted number (#9684) --- front/components/home/LandingLayout.tsx | 2 +- front/components/home/TrustedBy.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/front/components/home/LandingLayout.tsx b/front/components/home/LandingLayout.tsx index 223cff253143..711a9e37a0e6 100644 --- a/front/components/home/LandingLayout.tsx +++ b/front/components/home/LandingLayout.tsx @@ -101,7 +101,7 @@ export default function LandingLayout({