conversation actions: various fixes (#8855)

* conversation actions: various fixes

* parametric prompt

* more

front/lib/api/assistant/actions/conversation/include_file.ts

@@ -115,8 +115,10 @@ export class ConversationIncludeFileAction extends BaseAction {
     // that past actions on a previous version of a content fragment will correctly render the
     // content as being superseded showing the model that a new version available. The fileId of
     // that new version will be different but the title will likely be the same and the model should
-    // be able to undertstand the state of affair.
-    const m = (conversation.content.flat(1).find((m) => {
+    // be able to undertstand the state of affair. We use content.flat() to consider all versions of
+    // messages here (to support rendering a file that was part of an old version of a previous
+    // message).
+    const m = (conversation.content.flat().find((m) => {
       if (
         isContentFragmentType(m) &&
         isConversationIncludableFileContentType(m.contentType) &&
@@ -128,7 +130,9 @@ export class ConversationIncludeFileAction extends BaseAction {
     }) || null) as ContentFragmentType | null;
 
     if (!m) {
-      return new Err(`File \`${fileId}\` not found in conversation`);
+      return new Err(
+        `File \`${fileId}\` not includable or not found in conversation`
+      );
     }
 
     const rRes = await renderContentFragmentForModel(m, conversation, model, {

front/lib/api/assistant/actions/conversation/list_files.ts

@@ -9,7 +9,12 @@ import type {
 import { BaseAction } from "@dust-tt/types";
 import _ from "lodash";
 
-import { DEFAULT_CONVERSATION_LIST_FILES_ACTION_NAME } from "@app/lib/api/assistant/actions/constants";
+import {
+  DEFAULT_CONVERSATION_INCLUDE_FILE_ACTION_NAME,
+  DEFAULT_CONVERSATION_LIST_FILES_ACTION_NAME,
+  DEFAULT_CONVERSATION_QUERY_TABLES_ACTION_NAME,
+  DEFAULT_CONVERSATION_SEARCH_ACTION_NAME,
+} from "@app/lib/api/assistant/actions/constants";
 
 interface ConversationListFilesActionBlob {
   agentMessageId: ModelId;
@@ -46,10 +51,10 @@ export class ConversationListFilesAction extends BaseAction {
 
   async renderForMultiActionsModel(): Promise<FunctionMessageTypeModel> {
     let content =
-      `List of files attached to the conversation with their content type.\n\n` +
-      `- only the files marked as \`includable\` can be included with ` +
-      `the \`include_conversation_file\` tool.\n` +
-      // TODO(spolu): add mention of viz if enabled and other tools.
+      `List of files attached to the conversation with their content type and status (includable, queryable, searchable).\n\n` +
+      `// includable: can be rerieved with the \`${DEFAULT_CONVERSATION_INCLUDE_FILE_ACTION_NAME}\` action.\n` +
+      `// queryable: can be queried with the \`${DEFAULT_CONVERSATION_QUERY_TABLES_ACTION_NAME}\` action.\n` +
+      `// searchable: can be searched with the \`${DEFAULT_CONVERSATION_SEARCH_ACTION_NAME}\` action.\n` +
       `\n`;
     for (const f of this.files) {
       content += `<file id="${f.fileId}" name="${_.escape(f.title)}" type="${f.contentType}" includable="${f.isIncludable}" queryable="${f.isQueryable}" searchable="${f.isSearchable}"`;

front/lib/api/assistant/jit_actions.ts

@@ -127,7 +127,9 @@ export function listFiles(
   conversation: ConversationType
 ): ConversationFileType[] {
   const files: ConversationFileType[] = [];
-  for (const m of conversation.content.flat(1)) {
+  for (const versions of conversation.content) {
+    const m = versions[versions.length - 1];
+
     if (
       isContentFragmentType(m) &&
       isSupportedPlainTextContentType(m.contentType) &&

front/lib/api/assistant/visualization.ts

@@ -22,11 +22,13 @@ export async function getVisualizationPrompt({
   // If `jit_conversations_actions` is enabled we rely on the `conversations_list_files` emulated
   // actions to make the list of files available to the agent.
   if (await isJITActionsEnabled(auth)) {
-    return visualizationSystemPrompt;
+    return visualizationSystemPrompt(true);
   }
 
   const contentFragmentMessages: Array<ContentFragmentType> = [];
-  for (const m of conversation.content.flat(1)) {
+  for (const versions of conversation.content) {
+    const m = versions[versions.length - 1];
+
     if (isContentFragmentType(m)) {
       contentFragmentMessages.push(m);
     }
@@ -39,10 +41,12 @@ export async function getVisualizationPrompt({
     "sId"
   );
 
-  let prompt = visualizationSystemPrompt.trim() + "\n\n";
+  let prompt = visualizationSystemPrompt(false).trim() + "\n\n";
 
   const fileAttachments: string[] = [];
-  for (const m of conversation.content.flat(1)) {
+  for (const versions of conversation.content) {
+    const m = versions[versions.length - 1];
+
     if (isContentFragmentType(m)) {
       if (!m.fileId || !contentFragmentFileBySid[m.fileId]) {
         continue;
@@ -79,7 +83,7 @@ export async function getVisualizationPrompt({
   return prompt;
 }
 
-export const visualizationSystemPrompt = `\
+export const visualizationSystemPrompt = (jitActionsEnabled: boolean) => `\
 It is possible to generate visualizations for the user (using React components executed in a react-runner environment) that will be rendered in the user's browser by using the :::visualization container block markdown directive.
 
 Guidelines using the :::visualization tag:
@@ -108,8 +112,8 @@ Guidelines using the :::visualization tag:
   - Always use padding around plots to ensure elements are fully visible and labels/legends do not overlap with the plot or with each other.
   - Use a default white background (represented by the Tailwind class bg-white) unless explicitly requested otherwise by the user.
   - If you need to generate a legend for a chart, ensure it uses relative positioning or follows the natural flow of the layout, avoiding \`position: absolute\`, to maintain responsiveness and adaptability.
-- Using files from the conversation when available:
- - Files from the conversation can be accessed using the \`useFile()\` hook.
+- Using ${jitActionsEnabled ? "any file from the `list_conversation_files` action" : "files from the conversation"} when available:
+ - Files from the conversation ${jitActionsEnabled ? "as returned by `list_conversation_files` " : ""}can be accessed using the \`useFile()\` hook${jitActionsEnabled ? " (all files can be accessed by the hook irrespective of their status)" : ""}.
  - Once/if the file is available, \`useFile()\` will return a non-null \`File\` object. The \`File\` object is a browser File object. Examples of using \`useFile\` are available below.
  - Always use \`papaparse\` to parse CSV files.
  - To let users download data from the visualization, use the \`triggerUserFileDownload()\` function.
@@ -139,7 +143,7 @@ if (file) {
 }
 \`\`\`
 
-\`fileId\` can be extracted from the \`<file id="\${FILE_ID}" type... name...>\` tags in the conversation history.
+\`fileId\` can be extracted from the \`<file id="\${FILE_ID}" type... name...>\` tags ${jitActionsEnabled ? "returned by the `list_conversation_files` action" : "in the conversation history"}.
 
 Example using the \`triggerUserFileDownload\` hook: