[JIT Datasources] Create conversations space (#8662)

* create group by default & hide from ux * restriction on v1 routes * restriction on internal routes * unicity constraint * backfill * fix schema * clean * fix enum * flag conv space * fix * lint
dust-tt · Nov 15, 2024 · fb44741 · fb44741
1 parent 5daeb85
commit fb44741
Show file tree

Hide file tree

Showing 52 changed files with 558 additions and 31 deletions.
diff --git a/front/components/assistant_builder/spaces/SpaceSelector.tsx b/front/components/assistant_builder/spaces/SpaceSelector.tsx
@@ -8,7 +8,11 @@ import {
 import type { SpaceType } from "@dust-tt/types";
 import React, { useState } from "react";
 
-import { getSpaceIcon, getSpaceName, groupSpaces } from "@app/lib/spaces";
+import {
+  getSpaceIcon,
+  getSpaceName,
+  groupSpacesForDisplay,
+} from "@app/lib/spaces";
 import { classNames } from "@app/lib/utils";
 
 interface SpaceSelectorProps {
@@ -39,7 +43,7 @@ export function SpaceSelector({
   }
 
   // Group by kind and sort.
-  const sortedSpaces = groupSpaces(spaces)
+  const sortedSpaces = groupSpacesForDisplay(spaces)
     .filter((i) => i.section !== "system")
     .map((i) =>
       i.spaces.sort((a, b) => {

diff --git a/front/components/spaces/SpaceSideBarMenu.tsx b/front/components/spaces/SpaceSideBarMenu.tsx
@@ -28,7 +28,11 @@ import { getConnectorProviderLogoWithFallback } from "@app/lib/connector_provide
 import { getVisualForContentNode } from "@app/lib/content_nodes";
 import { getDataSourceNameFromView } from "@app/lib/data_sources";
 import type { SpaceSectionGroupType } from "@app/lib/spaces";
-import { getSpaceIcon, getSpaceName, groupSpaces } from "@app/lib/spaces";
+import {
+  getSpaceIcon,
+  getSpaceName,
+  groupSpacesForDisplay,
+} from "@app/lib/spaces";
 import { useApps } from "@app/lib/swr/apps";
 import { useDataSourceViewContentNodes } from "@app/lib/swr/data_source_views";
 import {
@@ -88,7 +92,7 @@ export default function SpaceSideBarMenu({
     return <></>;
   }
 
-  const sortedGroupedSpaces = groupSpaces(spaces).filter(
+  const sortedGroupedSpaces = groupSpacesForDisplay(spaces).filter(
     ({ section, spaces }) => section !== "system" || spaces.length !== 0
   );
 

diff --git a/front/lib/resources/space_resource.ts b/front/lib/resources/space_resource.ts
@@ -112,9 +112,21 @@ export class SpaceResource extends BaseResource<SpaceModel> {
         [globalGroup]
       ));
 
+    const conversationsSpace =
+      existingSpaces.find((s) => s.kind === "conversations") ||
+      (await SpaceResource.makeNew(
+        {
+          name: "Conversations",
+          kind: "conversations",
+          workspaceId: auth.getNonNullableWorkspace().id,
+        },
+        [globalGroup]
+      ));
+
     return {
       systemSpace,
       globalSpace,
+      conversationsSpace,
     };
   }
 
@@ -170,25 +182,39 @@ export class SpaceResource extends BaseResource<SpaceModel> {
   }
 
   static async listWorkspaceSpaces(
-    auth: Authenticator
+    auth: Authenticator,
+    options?: { includeConversationsSpace?: boolean }
   ): Promise<SpaceResource[]> {
     const spaces = await this.baseFetch(auth);
 
-    return spaces.filter((s) => s.canList(auth));
+    if (!options?.includeConversationsSpace) {
+      return spaces.filter((s) => !s.isConversations());
+    }
+    return spaces;
   }
 
   static async listWorkspaceSpacesAsMember(auth: Authenticator) {
     const spaces = await this.baseFetch(auth);
 
     // using canRead() as we know that only members can read spaces (but admins can list them)
-    return spaces.filter((s) => s.canList(auth) && s.canRead(auth));
+    // also, conversations space is not meant for members
+    return spaces.filter(
+      (s) => s.canList(auth) && s.canRead(auth) && !s.isConversations()
+    );
   }
 
-  static async listWorkspaceDefaultSpaces(auth: Authenticator) {
+  static async listWorkspaceDefaultSpaces(
+    auth: Authenticator,
+    options?: { includeConversationsSpace?: boolean }
+  ) {
     return this.baseFetch(auth, {
       where: {
         kind: {
-          [Op.in]: ["system", "global"],
+          [Op.in]: [
+            "system",
+            "global",
+            ...(options?.includeConversationsSpace ? ["conversations"] : []),
+          ],
         },
       },
     });
@@ -470,8 +496,8 @@ export class SpaceResource extends BaseResource<SpaceModel> {
       ];
     }
 
-    // Default Workspace space.
-    if (this.isGlobal()) {
+    // Default Workspace space and Conversations space.
+    if (this.isGlobal() || this.isConversations()) {
       return [
         {
           workspaceId: this.workspaceId,
@@ -567,6 +593,10 @@ export class SpaceResource extends BaseResource<SpaceModel> {
     return this.kind === "system";
   }
 
+  isConversations() {
+    return this.kind === "conversations";
+  }
+
   isRegular() {
     return this.kind === "regular";
   }

diff --git a/front/lib/resources/storage/models/spaces.ts b/front/lib/resources/storage/models/spaces.ts
@@ -1,5 +1,11 @@
 import type { SpaceKind } from "@dust-tt/types";
-import type { CreationOptional, ForeignKey, NonAttribute } from "sequelize";
+import { isUniqueSpaceKind } from "@dust-tt/types";
+import type {
+  CreationOptional,
+  ForeignKey,
+  NonAttribute,
+  Transaction,
+} from "sequelize";
 import { DataTypes } from "sequelize";
 
 import { Workspace } from "@app/lib/models/workspace";
@@ -66,3 +72,25 @@ SpaceModel.belongsTo(Workspace, {
   foreignKey: { allowNull: false },
   onDelete: "RESTRICT",
 });
+
+SpaceModel.addHook(
+  "beforeCreate",
+  "enforce_one_special_space_per_workspace",
+  async (space: SpaceModel, options: { transaction: Transaction }) => {
+    if (isUniqueSpaceKind(space.kind)) {
+      const existingSpace = await SpaceModel.findOne({
+        where: {
+          workspaceId: space.workspaceId,
+          kind: space.kind,
+        },
+        transaction: options.transaction,
+      });
+
+      if (existingSpace) {
+        throw new Error(`A ${space.kind} space exists for this workspace.`, {
+          cause: `enforce_one_${space.kind}_space_per_workspace`,
+        });
+      }
+    }
+  }
+);
diff --git a/front/lib/spaces.ts b/front/lib/spaces.ts
@@ -1,5 +1,6 @@
 import { LockIcon, PlanetIcon, ServerIcon } from "@dust-tt/sparkle";
 import type { PlanType, SpaceType, WorkspaceType } from "@dust-tt/types";
+import { assertNever } from "@dust-tt/types";
 import { groupBy } from "lodash";
 import type React from "react";
 
@@ -37,19 +38,34 @@ export const dustAppsListUrl = (
   return `/w/${owner.sId}/spaces/${space.sId}/categories/apps`;
 };
 
-export const groupSpaces = (spaces: SpaceType[]) => {
+export const groupSpacesForDisplay = (spaces: SpaceType[]) => {
+  // Conversations space should never be displayed
+  const spacesWithoutConversations = spaces.filter(
+    (space) => space.kind !== "conversations"
+  );
   // Group by kind and sort.
-  const groupedSpaces = groupBy(spaces, (space): SpaceSectionGroupType => {
-    switch (space.kind) {
-      case "public":
-      case "system":
-        return space.kind;
+  const groupedSpaces = groupBy(
+    spacesWithoutConversations,
+    (space): SpaceSectionGroupType => {
+      // please ts
+      if (space.kind === "conversations") {
+        throw new Error("Conversations space should never be displayed");
+      }
 
-      case "global":
-      case "regular":
-        return space.isRestricted ? "restricted" : "shared";
+      switch (space.kind) {
+        case "public":
+        case "system":
+          return space.kind;
+
+        case "global":
+        case "regular":
+          return space.isRestricted ? "restricted" : "shared";
+
+        default:
+          assertNever(space.kind);
+      }
     }
-  });
+  );
 
   return SPACE_SECTION_GROUP_ORDER.map((section) => ({
     section,

diff --git a/front/migrations/20241114_conversations_spaces_backfill.ts b/front/migrations/20241114_conversations_spaces_backfill.ts
@@ -0,0 +1,60 @@
+import _ from "lodash";
+
+import { Workspace } from "@app/lib/models/workspace";
+import { GroupResource } from "@app/lib/resources/group_resource";
+import { SpaceResource } from "@app/lib/resources/space_resource";
+import { makeScript } from "@app/scripts/helpers";
+
+async function backfillWorkspacesGroup(execute: boolean) {
+  const workspaces = await Workspace.findAll();
+
+  const chunks = _.chunk(workspaces, 16);
+  for (const [i, c] of chunks.entries()) {
+    console.log(
+      `[execute=${execute}] Processing chunk of ${c.length} workspaces... (${
+        i + 1
+      }/${chunks.length})`
+    );
+    if (execute) {
+      await Promise.all(
+        c.map((w) =>
+          (async () => {
+            try {
+              const workspaceGroup =
+                await GroupResource.internalFetchWorkspaceGlobalGroup(w.id);
+              if (!workspaceGroup) {
+                throw new Error("Workspace group not found");
+              }
+              await SpaceResource.makeNew(
+                {
+                  name: "Conversations",
+                  kind: "conversations",
+                  workspaceId: w.id,
+                },
+                [workspaceGroup]
+              );
+            } catch (error) {
+              if (
+                error instanceof Error &&
+                error.cause &&
+                error.cause === "enforce_one_conversations_space_per_workspace"
+              ) {
+                console.log(
+                  `Conversation already exists for workspace ${w.id}`
+                );
+              } else {
+                throw error;
+              }
+            }
+          })()
+        )
+      );
+    }
+  }
+
+  console.log(`Done.`);
+}
+
+makeScript({}, async ({ execute }) => {
+  await backfillWorkspacesGroup(execute);
+});
diff --git a/front/pages/api/v1/w/[wId]/spaces/[spaceId]/apps/[aId]/runs/[runId]/index.ts b/front/pages/api/v1/w/[wId]/spaces/[spaceId]/apps/[aId]/runs/[runId]/index.ts
@@ -95,6 +95,16 @@ async function handler(
     });
   }
 
+  if (app.space.kind === "conversations") {
+    return apiError(req, res, {
+      status_code: 404,
+      api_error: {
+        type: "space_not_found",
+        message: "The space you're trying to access was not found",
+      },
+    });
+  }
+
   switch (req.method) {
     case "GET":
       const runId = req.query.runId as string;

diff --git a/front/pages/api/v1/w/[wId]/spaces/[spaceId]/apps/[aId]/runs/index.ts b/front/pages/api/v1/w/[wId]/spaces/[spaceId]/apps/[aId]/runs/index.ts
@@ -207,6 +207,16 @@ async function handler(
     });
   }
 
+  if (app.space.kind === "conversations") {
+    return apiError(req, res, {
+      status_code: 404,
+      api_error: {
+        type: "space_not_found",
+        message: "The space you're trying to access was not found",
+      },
+    });
+  }
+
   if (!app.canRead(keyAuth)) {
     return apiError(req, res, {
       status_code: 403,

diff --git a/front/pages/api/v1/w/[wId]/spaces/[spaceId]/apps/index.ts b/front/pages/api/v1/w/[wId]/spaces/[spaceId]/apps/index.ts
@@ -104,6 +104,16 @@ async function handler(
     });
   }
 
+  if (space.kind === "conversations") {
+    return apiError(req, res, {
+      status_code: 404,
+      api_error: {
+        type: "space_not_found",
+        message: "The space you're trying to access was not found",
+      },
+    });
+  }
+
   switch (req.method) {
     case "GET":
       const apps = await AppResource.listBySpace(auth, space);

diff --git a/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_source_views/[dsvId]/index.ts b/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_source_views/[dsvId]/index.ts
@@ -186,6 +186,16 @@ async function handler(
     });
   }
 
+  if (dataSourceView.space.kind === "conversations") {
+    return apiError(req, res, {
+      status_code: 404,
+      api_error: {
+        type: "space_not_found",
+        message: "The space you're trying to access was not found",
+      },
+    });
+  }
+
   switch (req.method) {
     case "GET":
       return res.status(200).json({

diff --git a/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_source_views/[dsvId]/search.ts b/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_source_views/[dsvId]/search.ts
@@ -172,6 +172,16 @@ async function handler(
     });
   }
 
+  if (dataSourceView.space.kind === "conversations") {
+    return apiError(req, res, {
+      status_code: 404,
+      api_error: {
+        type: "space_not_found",
+        message: "The space you're trying to access was not found",
+      },
+    });
+  }
+
   switch (req.method) {
     case "GET": {
       // I could not find a way to make the query params be an array if there is only one tag.

diff --git a/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_source_views/index.ts b/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_source_views/index.ts
@@ -78,6 +78,16 @@ async function handler(
     });
   }
 
+  if (space.kind === "conversations") {
+    return apiError(req, res, {
+      status_code: 404,
+      api_error: {
+        type: "space_not_found",
+        message: "The space you're trying to access was not found",
+      },
+    });
+  }
+
   switch (req.method) {
     case "GET":
       const dataSourceViews = await DataSourceViewResource.listBySpace(