From c5c02036982483bf89491aba1af5369413219e42 Mon Sep 17 00:00:00 2001 From: Lucas Massemin Date: Tue, 3 Dec 2024 10:11:43 +0100 Subject: [PATCH] Allowed mimetype and title params for table creation (#9016) * Allowed mimetype and title params for table creation * Modified swagger to include new types, and have title and name always have the same value (title overrides) * Updated /tables post swagger * removed default mimeType value * Fixed callback deps warning * Enforce that table id is in parent in first position, even when provided * Allowing different values for title and names in api endpoints * tested and works as expected, removing forbidden field from swagger --------- Co-authored-by: Lucas --- .../data_source/TableUploadOrEditModal.tsx | 3 + .../data_sources/[dsId]/tables/[tId]/index.ts | 4 +- .../data_sources/[dsId]/tables/csv.ts | 6 +- .../data_sources/[dsId]/tables/index.ts | 18 ++- front/pages/api/v1/w/[wId]/swagger_schemas.ts | 61 ++++++++++ front/public/swagger.json | 107 +++++++++++++++++- sdks/js/src/types.ts | 2 + types/src/front/lib/core_api.ts | 2 + 8 files changed, 196 insertions(+), 7 deletions(-) diff --git a/front/components/data_source/TableUploadOrEditModal.tsx b/front/components/data_source/TableUploadOrEditModal.tsx index efe6fc274557..74d3cb7375a0 100644 --- a/front/components/data_source/TableUploadOrEditModal.tsx +++ b/front/components/data_source/TableUploadOrEditModal.tsx @@ -144,6 +144,8 @@ export const TableUploadOrEditModal = ({ truncate: true, async: false, useAppForHeaderDetection, + title: table.name, + mimeType: tableState.file?.type ?? "text/csv", }; let upsertRes = null; if (initialId) { @@ -184,6 +186,7 @@ export const TableUploadOrEditModal = ({ doUpdate, fileUploaderService, useAppForHeaderDetection, + tableState.file?.type, ] ); diff --git a/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/[tId]/index.ts b/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/[tId]/index.ts index 351544ff7c98..0cbf77a0d83d 100644 --- a/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/[tId]/index.ts +++ b/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/[tId]/index.ts @@ -53,7 +53,7 @@ import { apiError } from "@app/logger/withlogging"; * content: * application/json: * schema: - * $ref: '#/components/schemas/Datasource' + * $ref: '#/components/schemas/Table' * 404: * description: The table was not found * 405: @@ -205,6 +205,8 @@ async function handler( timestamp: table.timestamp, tags: table.tags, parents: table.parents, + mime_type: table.mime_type, + title: table.title, }, }); diff --git a/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/csv.ts b/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/csv.ts index c7935f8ddd3c..c6f702666ff0 100644 --- a/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/csv.ts +++ b/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/csv.ts @@ -114,7 +114,11 @@ async function handler( } const upsertRes = await handleDataSourceTableCSVUpsert({ auth, - params: r.data, + params: { + ...r.data, + mimeType: r.data.mimeType, + title: r.data.title ?? r.data.name, + }, dataSource, }); diff --git a/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/index.ts b/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/index.ts index e89cf2b119cc..58c4d64284ba 100644 --- a/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/index.ts +++ b/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/index.ts @@ -53,7 +53,7 @@ import { apiError } from "@app/logger/withlogging"; * schema: * type: array * items: - * $ref: '#/components/schemas/Datasource' + * $ref: '#/components/schemas/Table' * 400: * description: Invalid request * post: @@ -92,6 +92,9 @@ import { apiError } from "@app/logger/withlogging"; * name: * type: string * description: Name of the table + * title: + * type: string + * description: Title of the table * table_id: * type: string * description: Unique identifier for the table @@ -226,6 +229,8 @@ async function handler( timestamp: table.timestamp, tags: table.tags, parents: table.parents, + mime_type: table.mime_type, + title: table.title, }; }), }); @@ -345,6 +350,12 @@ async function handler( }); } + // Enforce that the table is a parent of itself by default. + const parentsWithTableId = + parents?.includes(tableId) && parents[0] === tableId + ? parents + : [tableId, ...(parents || []).filter((p) => p !== tableId)]; + const upsertRes = await coreAPI.upsertTable({ projectId: dataSource.dustAPIProjectId, dataSourceId: dataSource.dustAPIDataSourceId, @@ -353,7 +364,8 @@ async function handler( description, timestamp: timestamp ?? null, tags: tags || [], - parents: parents || [], + // Table is a parent of itself by default. + parents: parentsWithTableId, remoteDatabaseTableId: remoteDatabaseTableId ?? null, remoteDatabaseSecretId: remoteDatabaseSecretId ?? null, title, @@ -394,6 +406,8 @@ async function handler( timestamp: table.timestamp, tags: table.tags, parents: table.parents, + mime_type: table.mime_type, + title: table.title, }, }); diff --git a/front/pages/api/v1/w/[wId]/swagger_schemas.ts b/front/pages/api/v1/w/[wId]/swagger_schemas.ts index 30938d6ffccf..a350e3a40072 100644 --- a/front/pages/api/v1/w/[wId]/swagger_schemas.ts +++ b/front/pages/api/v1/w/[wId]/swagger_schemas.ts @@ -414,6 +414,67 @@ * type: boolean * description: Whether this datasource is selected by default for assistants * example: true + * Table: + * type: object + * properties: + * name: + * type: string + * description: Name of the table + * example: "Roi data" + * deprecated: true + * title: + * type: string + * description: Title of the table + * example: "ROI Data" + * table_id: + * type: string + * description: Unique identifier for the table + * example: "1234f4567c" + * description: + * type: string + * description: Description of the table + * example: "roi data for Q1" + * mime_type: + * type: string + * description: MIME type of the table + * example: "text/csv" + * schema: + * type: array + * description: Array of column definitions + * items: + * type: object + * properties: + * name: + * type: string + * description: Name of the column + * example: "roi" + * value_type: + * type: string + * description: Data type of the column + * enum: [text, int, float, bool, date] + * example: "int" + * possible_values: + * type: array + * description: Array of possible values for the column (null if unrestricted) + * items: + * type: string + * nullable: true + * example: ["1", "2", "3"] + * timestamp: + * type: number + * description: Unix timestamp of table creation/modification + * example: 1732810375150 + * tags: + * type: array + * description: Array of tags associated with the table + * items: + * type: string + * parents: + * type: array + * description: Array of parent table IDs + * items: + * type: string + * example: ["1234f4567c"] * DatasourceView: * type: object * properties: diff --git a/front/public/swagger.json b/front/public/swagger.json index d0f9457cef87..db08f0606570 100644 --- a/front/public/swagger.json +++ b/front/public/swagger.json @@ -2571,7 +2571,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Datasource" + "$ref": "#/components/schemas/Table" } } } @@ -3065,7 +3065,7 @@ "schema": { "type": "array", "items": { - "$ref": "#/components/schemas/Datasource" + "$ref": "#/components/schemas/Table" } } } @@ -3125,7 +3125,16 @@ "properties": { "name": { "type": "string", - "description": "Name of the table" + "description": "Name of the table", + "deprecated": true + }, + "title": { + "type": "string", + "description": "Title of the table" + }, + "mime_type": { + "type": "string", + "description": "Mime type of the table" }, "table_id": { "type": "string", @@ -3957,6 +3966,98 @@ } } }, + "Table": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Name of the table", + "example": "Roi data", + "deprecated": true + }, + "title": { + "type": "string", + "description": "Title of the table", + "example": "ROI Data" + }, + "table_id": { + "type": "string", + "description": "Unique identifier for the table", + "example": "1234f4567c" + }, + "description": { + "type": "string", + "description": "Description of the table", + "example": "roi data for Q1" + }, + "mime_type": { + "type": "string", + "description": "MIME type of the table", + "example": "text/csv" + }, + "schema": { + "type": "array", + "description": "Array of column definitions", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Name of the column", + "example": "roi" + }, + "value_type": { + "type": "string", + "description": "Data type of the column", + "enum": [ + "text", + "int", + "float", + "bool", + "date" + ], + "example": "int" + }, + "possible_values": { + "type": "array", + "description": "Array of possible values for the column (null if unrestricted)", + "items": { + "type": "string" + }, + "nullable": true, + "example": [ + "1", + "2", + "3" + ] + } + } + } + }, + "timestamp": { + "type": "number", + "description": "Unix timestamp of table creation/modification", + "example": 1732810375150 + }, + "tags": { + "type": "array", + "description": "Array of tags associated with the table", + "items": { + "type": "string" + } + }, + "parents": { + "type": "array", + "description": "Array of parent table IDs", + "items": { + "type": "string" + }, + "example": [ + "1234f4567c" + ] + } + } + }, "DatasourceView": { "type": "object", "properties": { diff --git a/sdks/js/src/types.ts b/sdks/js/src/types.ts index 8c8ee563b8cc..cc7172caf81a 100644 --- a/sdks/js/src/types.ts +++ b/sdks/js/src/types.ts @@ -314,6 +314,8 @@ const CoreAPITablePublicSchema = z.object({ timestamp: z.number(), tags: z.array(z.string()), parents: z.array(z.string()), + mime_type: z.string().optional(), + title: z.string().optional(), }); export type CoreAPITablePublic = z.infer; diff --git a/types/src/front/lib/core_api.ts b/types/src/front/lib/core_api.ts index 33c6099aa9d7..d0ec85ae1bf3 100644 --- a/types/src/front/lib/core_api.ts +++ b/types/src/front/lib/core_api.ts @@ -128,6 +128,8 @@ export type CoreAPITable = { parents: string[]; created: number; data_source_id: string; + title: string; + mime_type: string; remote_database_table_id: string | null; remote_database_secret_id: string | null; };