add a magical value to hide content nodes

dust-tt · Dec 13, 2024 · 755e673 · 755e673
1 parent 6ddd02c
commit 755e673
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 15 deletions.
diff --git a/connectors/src/connectors/confluence/temporal/activities.ts b/connectors/src/connectors/confluence/temporal/activities.ts
@@ -18,10 +18,7 @@ import {
   getConfluencePageParentIds,
   getSpaceHierarchy,
 } from "@connectors/connectors/confluence/lib/hierarchy";
-import {
-  makePageInternalId,
-  makeSpaceInternalId,
-} from "@connectors/connectors/confluence/lib/internal_ids";
+import { makePageInternalId } from "@connectors/connectors/confluence/lib/internal_ids";
 import { makeConfluenceDocumentUrl } from "@connectors/connectors/confluence/temporal/workflow_ids";
 import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config";
 import { concurrentExecutor } from "@connectors/lib/async_utils";
@@ -376,7 +373,6 @@ export async function confluenceCheckAndUpsertPageActivity({
       `version:${page.version.number}`,
       ...customTags,
     ];
-    const spaceId = makeSpaceInternalId(page.spaceId);
 
     await upsertToDatasource({
       dataSourceConfig,
@@ -385,8 +381,8 @@ export async function confluenceCheckAndUpsertPageActivity({
       documentUrl,
       loggerArgs,
       // Parent Ids will be computed after all page imports within the space have been completed.
-      parents: [documentId, spaceId],
-      parentId: spaceId,
+      parents: [documentId],
+      hideContentNode: true,
       tags,
       timestampMs: lastPageVersionCreatedAt.getTime(),
       upsertContext: {

diff --git a/connectors/src/lib/data_sources.ts b/connectors/src/lib/data_sources.ts
@@ -9,6 +9,7 @@ import type {
   PostDataSourceDocumentRequestBody,
 } from "@dust-tt/types";
 import {
+  HiddenContentNodeParentId,
   isValidDate,
   MAX_CHUNK_SIZE,
   safeSubstring,
@@ -69,13 +70,15 @@ type UpsertToDataSourceParams = {
   timestampMs?: number;
   tags?: string[];
   parents: string[];
-  parentId?: string | null;
   loggerArgs?: Record<string, string | number>;
   upsertContext: UpsertContext;
   title: string;
   mimeType: string;
   async: boolean;
-};
+} & (
+  | { parentId: string; hideContentNode?: never }
+  | { parentId?: never; hideContentNode: true }
+);
 
 function getDustAPI(dataSourceConfig: DataSourceConfig) {
   return new DustAPI(
@@ -104,7 +107,8 @@ async function _upsertToDatasource({
   title,
   mimeType,
   async,
-  parentId = null,
+  parentId,
+  hideContentNode,
 }: UpsertToDataSourceParams) {
   return tracer.trace(
     `connectors`,
@@ -150,6 +154,15 @@ async function _upsertToDatasource({
         ? (Math.floor(timestampMs) as Branded<number, IntBrand>)
         : null;
 
+      // setting the magic values to hide the content node
+      if (hideContentNode) {
+        parentId = HiddenContentNodeParentId; // types enforce that we don't already have a parentId there
+        parents = [
+          ...parents.filter((p) => p !== HiddenContentNodeParentId),
+          HiddenContentNodeParentId,
+        ];
+      }
+
       const dustRequestPayload: PostDataSourceDocumentRequestBody = {
         text: null,
         section: documentContent,

diff --git a/front/components/ContentNodeTree.tsx b/front/components/ContentNodeTree.tsx
@@ -9,6 +9,7 @@ import {
   Tree,
 } from "@dust-tt/sparkle";
 import type { APIError, BaseContentNode } from "@dust-tt/types";
+import { isContentNodeHidden } from "@dust-tt/types";
 import type { ReactNode } from "react";
 import React, { useCallback, useContext, useState } from "react";
 
@@ -117,7 +118,9 @@ function ContentNodeTreeChildren({
     useResourcesHook(parentId);
 
   const filteredNodes = resources.filter(
-    (n) => search.trim().length === 0 || n.title.includes(search)
+    (n) =>
+      (search.trim().length === 0 || n.title.includes(search)) &&
+      !isContentNodeHidden(n)
   );
 
   const getCheckedState = useCallback(

diff --git a/types/src/front/lib/connectors_api.ts b/types/src/front/lib/connectors_api.ts
@@ -1,7 +1,4 @@
-import {
-  AdminCommandType,
-  AdminResponseType,
-} from "../../connectors/admin/cli";
+import { AdminCommandType, AdminResponseType } from "../../connectors/admin/cli";
 import { ConnectorsAPIError, isConnectorsAPIError } from "../../connectors/api";
 import { UpdateConnectorConfigurationType } from "../../connectors/api_handlers/connector_configuration";
 import { ConnectorCreateRequestBody } from "../../connectors/api_handlers/create_connector";
@@ -121,6 +118,16 @@ export type ContentNodeWithParentIds = ContentNode & {
   parentInternalIds: string[] | null;
 };
 
+/**
+ * This type represents the ID that should be passed as parentId to a content node to hide it from the UI.
+ * This behavior is typically used to hide content nodes whose position in the ContentNodeTree cannot be resolved at time of upsertion.
+ */
+export const HiddenContentNodeParentId = "__dust_synchronizing__";
+
+export function isContentNodeHidden(contentNode: BaseContentNode): boolean {
+  return contentNode.parentInternalId === HiddenContentNodeParentId;
+}
+
 type GetContentNodesReturnType<
   IncludeParents extends boolean,
   Key extends string