diff --git a/front/admin/cli.ts b/front/admin/cli.ts index c02d81828a8b..250d535f9f5d 100644 --- a/front/admin/cli.ts +++ b/front/admin/cli.ts @@ -4,9 +4,7 @@ import { SUPPORTED_MODEL_CONFIGS, } from "@dust-tt/types"; import { CoreAPI } from "@dust-tt/types"; -import { Storage } from "@google-cloud/storage"; import parseArgs from "minimist"; -import readline from "readline"; import { getConversation } from "@app/lib/api/assistant/conversation"; import { @@ -265,114 +263,6 @@ const user = async (command: string, args: parseArgs.ParsedArgs) => { const dataSource = async (command: string, args: parseArgs.ParsedArgs) => { switch (command) { - case "delete": { - if (!args.wId) { - throw new Error("Missing --wId argument"); - } - if (!args.dsId) { - throw new Error("Missing --dsId argument"); - } - - const auth = await Authenticator.internalAdminForWorkspace(args.wId); - - const dataSource = await DataSourceResource.fetchById(auth, args.dsId); - if (!dataSource) { - throw new Error( - `DataSource not found: wId='${args.wId}' dsId='${args.dsId}'` - ); - } - - const dustAPIProjectId = dataSource.dustAPIProjectId; - - await new Promise((resolve) => { - const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout, - }); - rl.question( - `Are you sure you want to definitely delete the following data source and all associated data: wId='${args.wId}' dsId='${args.dsId}' provider='${dataSource.connectorProvider}'? (y/N) `, - (answer: string) => { - rl.close(); - if (answer !== "y") { - throw new Error("Aborting"); - } - resolve(null); - } - ); - }); - - if (dataSource.connectorId) { - console.log(`Deleting connectorId=${dataSource.connectorId}}`); - const connDeleteRes = await new ConnectorsAPI( - config.getConnectorsAPIConfig(), - logger - ).deleteConnector(dataSource.connectorId.toString(), true); - if (connDeleteRes.isErr()) { - throw new Error(connDeleteRes.error.message); - } - } - const coreAPI = new CoreAPI(config.getCoreAPIConfig(), logger); - - const coreDeleteRes = await coreAPI.deleteDataSource({ - projectId: dataSource.dustAPIProjectId, - dataSourceId: dataSource.dustAPIDataSourceId, - }); - if (coreDeleteRes.isErr()) { - throw new Error(coreDeleteRes.error.message); - } - - await dataSource.delete(auth); - - console.log("Data source deleted. Make sure to run: \n\n"); - console.log( - "\x1b[32m%s\x1b[0m", - `./admin/cli.sh data-source scrub --dustAPIProjectId ${dustAPIProjectId}` - ); - console.log( - "\n\n...to fully scrub the customer data from our infra (GCS clean-up)." - ); - console.log(`WARNING: For Github datasource, the user may want to uninstall the app from Github - to revoke the authorization. If needed, send an email (cf template in lib/email.ts) `); - return; - } - - case "scrub": { - if (!args.dustAPIProjectId) { - throw new Error("Missing --dustAPIProjectId argument"); - } - - const storage = new Storage({ keyFilename: config.getServiceAccount() }); - - const [files] = await storage - .bucket(config.getDustDataSourcesBucket()) - .getFiles({ prefix: `${args.dustAPIProjectId}` }); - - console.log(`Chunking ${files.length} files...`); - const chunkSize = 32; - const chunks = []; - for (let i = 0; i < files.length; i += chunkSize) { - chunks.push(files.slice(i, i + chunkSize)); - } - - for (let i = 0; i < chunks.length; i++) { - console.log(`Processing chunk ${i}/${chunks.length}...`); - const chunk = chunks[i]; - if (!chunk) { - continue; - } - await Promise.all( - chunk.map((f) => { - return (async () => { - console.log(`Deleting file: ${f.name}`); - await f.delete(); - })(); - }) - ); - } - - return; - } - case "delete-document": { if (!args.wId) { throw new Error("Missing --wId argument"); diff --git a/front/lib/api/data_sources.ts b/front/lib/api/data_sources.ts index f1a0ccf930e1..f63f96f37e42 100644 --- a/front/lib/api/data_sources.ts +++ b/front/lib/api/data_sources.ts @@ -54,7 +54,10 @@ export async function getDataSources( }); } -export async function deleteDataSource( +/** + * Soft delete a data source. This will mark the data source as deleted and will trigger a scrubbing. + */ +export async function softDeleteDataSourceAndLaunchScrubWorkflow( auth: Authenticator, dataSource: DataSourceResource, transaction?: Transaction @@ -70,8 +73,30 @@ export async function deleteDataSource( }); } - const dustAPIProjectId = dataSource.dustAPIProjectId; + await dataSource.delete(auth, { transaction, hardDelete: false }); + // The scrubbing workflow will delete associated resources and hard delete the data source. + await launchScrubDataSourceWorkflow(owner, dataSource); + + return new Ok(dataSource.toJSON()); +} + +/** + * Performs a hard deletion of the specified data source, ensuring complete removal of the data + * source and all its associated resources, including any existing connectors. + */ +export async function hardDeleteDataSource( + auth: Authenticator, + dataSource: DataSourceResource +) { + if (!auth.isBuilder()) { + return new Err({ + code: "unauthorized_deletion", + message: "Only builders can destroy data sources.", + }); + } + + const { dustAPIProjectId } = dataSource; if (dataSource.connectorId && dataSource.connectorProvider) { if ( !MANAGED_DS_DELETABLE.includes(dataSource.connectorProvider) && @@ -119,12 +144,8 @@ export async function deleteDataSource( } } - await dataSource.delete(auth, transaction); + await dataSource.delete(auth, { hardDelete: true }); - await launchScrubDataSourceWorkflow({ - wId: owner.sId, - dustAPIProjectId, - }); if (dataSource.connectorProvider) { await warnPostDeletion(auth, dataSource.connectorProvider); } diff --git a/front/lib/api/vaults.ts b/front/lib/api/vaults.ts index 01d575864115..5b5ce301e180 100644 --- a/front/lib/api/vaults.ts +++ b/front/lib/api/vaults.ts @@ -1,7 +1,7 @@ import type { DataSourceWithAgentsUsageType } from "@dust-tt/types"; import { uniq } from "lodash"; -import { deleteDataSource } from "@app/lib/api/data_sources"; +import { softDeleteDataSourceAndLaunchScrubWorkflow } from "@app/lib/api/data_sources"; import type { Authenticator } from "@app/lib/auth"; import { DataSourceResource } from "@app/lib/resources/data_source_resource"; import { DataSourceViewResource } from "@app/lib/resources/data_source_view_resource"; @@ -51,14 +51,18 @@ export const deleteVault = async ( await frontSequelize.transaction(async (t) => { // delete all data source views for (const view of dataSourceViews) { - const res = await view.delete(auth, t); + // Soft delete view, they will be hard deleted when the data source scrubbing job runs. + const res = await view.delete(auth, { + transaction: t, + hardDelete: false, + }); if (res.isErr()) { throw res.error; } } for (const ds of dataSources) { - const res = await deleteDataSource(auth, ds, t); + const res = await softDeleteDataSourceAndLaunchScrubWorkflow(auth, ds, t); if (res.isErr()) { throw res.error; } @@ -66,14 +70,14 @@ export const deleteVault = async ( // delete all vaults groups for (const group of vault.groups) { - const res = await group.delete(auth, t); + const res = await group.delete(auth, { transaction: t }); if (res.isErr()) { throw res.error; } } // Finally, delete the vault - const res = await vault.delete(auth, t); + const res = await vault.delete(auth, { transaction: t }); if (res.isErr()) { throw res.error; } diff --git a/front/lib/resources/app_resource.ts b/front/lib/resources/app_resource.ts index e7bce9536b95..be9704623174 100644 --- a/front/lib/resources/app_resource.ts +++ b/front/lib/resources/app_resource.ts @@ -150,29 +150,53 @@ export class AppResource extends ResourceWithVault { // Deletion. - async delete(auth: Authenticator): Promise> { - try { - await frontSequelize.transaction(async (t) => { - await RunResource.deleteAllByAppId(this.id, t); - await Clone.destroy({ - where: { - [Op.or]: [{ fromId: this.id }, { toId: this.id }], - }, - transaction: t, - }); - const res = await DatasetResource.deleteForApp(auth, this, t); - if (res.isErr()) { - // Interrupt the transaction if there was an error deleting datasets. - throw res.error; - } - await this.model.destroy({ - where: { - workspaceId: auth.getNonNullableWorkspace().id, - id: this.id, - }, - transaction: t, - }); + protected async hardDelete( + auth: Authenticator + ): Promise> { + const deletedCount = await frontSequelize.transaction(async (t) => { + await RunResource.deleteAllByAppId(this.id, t); + + await Clone.destroy({ + where: { + [Op.or]: [{ fromId: this.id }, { toId: this.id }], + }, + transaction: t, + }); + const res = await DatasetResource.deleteForApp(auth, this, t); + if (res.isErr()) { + // Interrupt the transaction if there was an error deleting datasets. + throw res.error; + } + + return App.destroy({ + where: { + workspaceId: auth.getNonNullableWorkspace().id, + id: this.id, + }, + transaction: t, + // Use 'hardDelete: true' to ensure the record is permanently deleted from the database, + // bypassing the soft deletion in place. + hardDelete: true, }); + }); + + return new Ok(deletedCount); + } + + // TODO(2024-09-27 flav): Implement soft delete of apps. + protected softDelete(): Promise> { + throw new Error("Method not implemented."); + } + + // TODO(2024-09-27 flav): Implement soft delete of apps. + async delete( + auth: Authenticator, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + { hardDelete }: { hardDelete: true } + ): Promise> { + try { + await this.hardDelete(auth); + return new Ok(undefined); } catch (err) { return new Err(err as Error); diff --git a/front/lib/resources/base_resource.ts b/front/lib/resources/base_resource.ts index e8db9dc3df1d..24f3415d5368 100644 --- a/front/lib/resources/base_resource.ts +++ b/front/lib/resources/base_resource.ts @@ -90,6 +90,6 @@ export abstract class BaseResource { abstract delete( auth: Authenticator, - transaction?: Transaction - ): Promise>; + { transaction }: { transaction?: Transaction } + ): Promise>; } diff --git a/front/lib/resources/content_fragment_resource.ts b/front/lib/resources/content_fragment_resource.ts index f4a3762669ff..f4e9394b3412 100644 --- a/front/lib/resources/content_fragment_resource.ts +++ b/front/lib/resources/content_fragment_resource.ts @@ -104,12 +104,7 @@ export class ContentFragmentResource extends BaseResource * Temporary workaround until we can call this method from the MessageResource. * @deprecated use the destroy method. */ - delete( - // eslint-disable-next-line @typescript-eslint/no-unused-vars - auth: Authenticator, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - transaction?: Transaction - ): Promise> { + delete(): Promise> { throw new Error("Method not implemented."); } diff --git a/front/lib/resources/data_source_resource.ts b/front/lib/resources/data_source_resource.ts index 655d48f18b8b..ed8e27cca8c5 100644 --- a/front/lib/resources/data_source_resource.ts +++ b/front/lib/resources/data_source_resource.ts @@ -5,7 +5,7 @@ import type { PokeDataSourceType, Result, } from "@dust-tt/types"; -import { Err, formatUserFullName, Ok, removeNulls } from "@dust-tt/types"; +import { formatUserFullName, Ok, removeNulls } from "@dust-tt/types"; import type { Attributes, CreationAttributes, @@ -50,6 +50,7 @@ export type FetchDataSourceOrigin = | "v1_data_sources_tokenize"; export type FetchDataSourceOptions = { + includeDeleted?: boolean; includeEditedBy?: boolean; limit?: number; order?: [string, "ASC" | "DESC"][]; @@ -132,9 +133,12 @@ export class DataSourceResource extends ResourceWithVault { fetchDataSourceOptions?: FetchDataSourceOptions, options?: ResourceFindOptions ) { + const { includeDeleted } = fetchDataSourceOptions ?? {}; + return this.baseFetchWithAuthorization(auth, { ...this.getOptions(fetchDataSourceOptions), ...options, + includeDeleted, }); } @@ -358,10 +362,34 @@ export class DataSourceResource extends ResourceWithVault { return dataSource ?? null; } - async delete( + protected async softDelete( + auth: Authenticator, + transaction?: Transaction + ): Promise> { + // Directly delete the DataSourceViewModel here to avoid a circular dependency. + await DataSourceViewModel.destroy({ + where: { + workspaceId: auth.getNonNullableWorkspace().id, + dataSourceId: this.id, + }, + transaction, + hardDelete: false, + }); + + const deletedCount = await this.model.destroy({ + where: { + id: this.id, + }, + transaction, + }); + + return new Ok(deletedCount); + } + + protected async hardDelete( auth: Authenticator, transaction?: Transaction - ): Promise> { + ): Promise> { await AgentDataSourceConfiguration.destroy({ where: { dataSourceId: this.id, @@ -376,28 +404,29 @@ export class DataSourceResource extends ResourceWithVault { transaction, }); - // We directly delete the DataSourceViewResource.model here to avoid a circular dependency. - // DataSourceViewResource depends on DataSourceResource + // Directly delete the DataSourceViewModel here to avoid a circular dependency. await DataSourceViewModel.destroy({ where: { workspaceId: auth.getNonNullableWorkspace().id, dataSourceId: this.id, }, transaction, + // Use 'hardDelete: true' to ensure the record is permanently deleted from the database, + // bypassing the soft deletion in place. + hardDelete: true, }); - try { - await this.model.destroy({ - where: { - id: this.id, - }, - transaction, - }); + const deletedCount = await DataSourceModel.destroy({ + where: { + id: this.id, + }, + transaction, + // Use 'hardDelete: true' to ensure the record is permanently deleted from the database, + // bypassing the soft deletion in place. + hardDelete: true, + }); - return new Ok(undefined); - } catch (err) { - return new Err(err as Error); - } + return new Ok(deletedCount); } // Updating. diff --git a/front/lib/resources/data_source_view_resource.ts b/front/lib/resources/data_source_view_resource.ts index 9e37818e43c7..c0b02f7d3c80 100644 --- a/front/lib/resources/data_source_view_resource.ts +++ b/front/lib/resources/data_source_view_resource.ts @@ -8,7 +8,7 @@ import type { PokeDataSourceViewType, Result, } from "@dust-tt/types"; -import { Err, formatUserFullName, Ok, removeNulls } from "@dust-tt/types"; +import { formatUserFullName, Ok, removeNulls } from "@dust-tt/types"; import type { Attributes, CreationAttributes, @@ -54,6 +54,7 @@ const getDataSourceCategory = ( }; export type FetchDataSourceViewOptions = { + includeDeleted?: boolean; includeEditedBy?: boolean; limit?: number; order?: [string, "ASC" | "DESC"][]; @@ -204,9 +205,12 @@ export class DataSourceViewResource extends ResourceWithVault ) { + const { includeDeleted } = fetchDataSourceViewOptions ?? {}; + const dataSourceViews = await this.baseFetchWithAuthorization(auth, { ...this.getOptions(fetchDataSourceViewOptions), ...options, + includeDeleted, }); const dataSourceIds = removeNulls( @@ -387,36 +391,51 @@ export class DataSourceViewResource extends ResourceWithVault> { - try { - // Delete agent configurations elements pointing to this data source view. - await AgentDataSourceConfiguration.destroy({ - where: { - dataSourceViewId: this.id, - }, - transaction, - }); - await AgentTablesQueryConfigurationTable.destroy({ - where: { - dataSourceViewId: this.id, - }, - }); + ): Promise> { + const deletedCount = await DataSourceViewModel.destroy({ + where: { + workspaceId: auth.getNonNullableWorkspace().id, + id: this.id, + }, + transaction, + hardDelete: false, + }); - await this.model.destroy({ - where: { - workspaceId: auth.getNonNullableWorkspace().id, - id: this.id, - }, - transaction, - }); + return new Ok(deletedCount); + } - return new Ok(undefined); - } catch (err) { - return new Err(err as Error); - } + async hardDelete( + auth: Authenticator, + transaction?: Transaction + ): Promise> { + // Delete agent configurations elements pointing to this data source view. + await AgentDataSourceConfiguration.destroy({ + where: { + dataSourceViewId: this.id, + }, + transaction, + }); + await AgentTablesQueryConfigurationTable.destroy({ + where: { + dataSourceViewId: this.id, + }, + }); + + const deletedCount = await DataSourceViewModel.destroy({ + where: { + workspaceId: auth.getNonNullableWorkspace().id, + id: this.id, + }, + transaction, + // Use 'hardDelete: true' to ensure the record is permanently deleted from the database, + // bypassing the soft deletion in place. + hardDelete: true, + }); + + return new Ok(deletedCount); } // This method can only be used once all agent configurations have been deleted. Otherwise use the @@ -425,11 +444,14 @@ export class DataSourceViewResource extends ResourceWithVault { async delete( auth: Authenticator, - t?: Transaction + { transaction }: { transaction?: Transaction } = {} ): Promise> { await Dataset.destroy({ where: { id: this.id, workspaceId: auth.getNonNullableWorkspace().id, }, - transaction: t, + transaction, }); return new Ok(undefined); } diff --git a/front/lib/resources/group_resource.ts b/front/lib/resources/group_resource.ts index 68b70e7990b8..e2d99301f774 100644 --- a/front/lib/resources/group_resource.ts +++ b/front/lib/resources/group_resource.ts @@ -618,7 +618,7 @@ export class GroupResource extends BaseResource { async delete( auth: Authenticator, - transaction?: Transaction + { transaction }: { transaction?: Transaction } = {} ): Promise> { try { await GroupVaultModel.destroy({ diff --git a/front/lib/resources/labs_transcripts_resource.ts b/front/lib/resources/labs_transcripts_resource.ts index df40622e592c..3cc302b00ee6 100644 --- a/front/lib/resources/labs_transcripts_resource.ts +++ b/front/lib/resources/labs_transcripts_resource.ts @@ -147,7 +147,7 @@ export class LabsTranscriptsConfigurationResource extends BaseResource> { try { await this.deleteHistory(transaction); diff --git a/front/lib/resources/membership_resource.ts b/front/lib/resources/membership_resource.ts index 601ac6fb9acf..64fd64622552 100644 --- a/front/lib/resources/membership_resource.ts +++ b/front/lib/resources/membership_resource.ts @@ -512,7 +512,7 @@ export class MembershipResource extends BaseResource { async delete( auth: Authenticator, - transaction?: Transaction + { transaction }: { transaction?: Transaction } ): Promise> { try { await this.model.destroy({ diff --git a/front/lib/resources/resource_with_vault.ts b/front/lib/resources/resource_with_vault.ts index 1351c2f960ff..05b8b098fbd8 100644 --- a/front/lib/resources/resource_with_vault.ts +++ b/front/lib/resources/resource_with_vault.ts @@ -1,9 +1,10 @@ +import type { Result } from "@dust-tt/types"; import type { Attributes, ForeignKey, Includeable, - ModelStatic, NonAttribute, + Transaction, WhereOptions, } from "sequelize"; import { Model } from "sequelize"; @@ -15,6 +16,10 @@ import { BaseResource } from "@app/lib/resources/base_resource"; import { GroupResource } from "@app/lib/resources/group_resource"; import { GroupModel } from "@app/lib/resources/storage/models/groups"; import type { VaultModel } from "@app/lib/resources/storage/models/vaults"; +import type { + ModelStaticSoftDeletable, + SoftDeletableModel, +} from "@app/lib/resources/storage/wrappers"; import type { InferIncludeType, ResourceFindOptions, @@ -29,12 +34,12 @@ interface ModelWithVault extends ResourceWithId { } export abstract class ResourceWithVault< - M extends Model & ModelWithVault, + M extends SoftDeletableModel & ModelWithVault, > extends BaseResource { readonly workspaceId: ModelWithVault["workspaceId"]; protected constructor( - model: ModelStatic, + model: ModelStaticSoftDeletable, blob: Attributes, public readonly vault: VaultResource ) { @@ -45,19 +50,25 @@ export abstract class ResourceWithVault< protected static async baseFetchWithAuthorization< T extends ResourceWithVault, - M extends Model & ModelWithVault, + M extends SoftDeletableModel & ModelWithVault, IncludeType extends Partial>, >( this: { new ( - model: ModelStatic, + model: ModelStaticSoftDeletable, blob: Attributes, vault: VaultResource, includes?: IncludeType ): T; - } & { model: ModelStatic }, + } & { model: ModelStaticSoftDeletable }, auth: Authenticator, - { includes, limit, order, where }: ResourceFindOptions = {} + { + includes, + limit, + order, + where, + includeDeleted, + }: ResourceFindOptions = {} ): Promise { const includeClauses: Includeable[] = [ { @@ -73,6 +84,7 @@ export abstract class ResourceWithVault< include: includeClauses, limit, order, + includeDeleted, }); return ( @@ -114,6 +126,31 @@ export abstract class ResourceWithVault< ); } + // Delete. + + protected abstract hardDelete( + auth: Authenticator, + transaction?: Transaction + ): Promise>; + + protected abstract softDelete( + auth: Authenticator, + transaction?: Transaction + ): Promise>; + + async delete( + auth: Authenticator, + options: { hardDelete: boolean; transaction?: Transaction } + ): Promise> { + const { hardDelete, transaction } = options; + + if (hardDelete) { + return this.hardDelete(auth, transaction); + } + + return this.softDelete(auth, transaction); + } + // Permissions. acl() { diff --git a/front/lib/resources/run_resource.ts b/front/lib/resources/run_resource.ts index 4e7e2cb805ff..dea0c1d27227 100644 --- a/front/lib/resources/run_resource.ts +++ b/front/lib/resources/run_resource.ts @@ -120,7 +120,7 @@ export class RunResource extends BaseResource { async delete( auth: Authenticator, - transaction?: Transaction + { transaction }: { transaction?: Transaction } = {} ): Promise> { try { // Delete the run usage entry. diff --git a/front/lib/resources/storage/models/apps.ts b/front/lib/resources/storage/models/apps.ts index f90340b63e8b..c65ad052a5e1 100644 --- a/front/lib/resources/storage/models/apps.ts +++ b/front/lib/resources/storage/models/apps.ts @@ -11,11 +11,9 @@ import { DataTypes, Model } from "sequelize"; import { Workspace } from "@app/lib/models/workspace"; import { frontSequelize } from "@app/lib/resources/storage"; import { VaultModel } from "@app/lib/resources/storage/models/vaults"; +import { SoftDeletableModel } from "@app/lib/resources/storage/wrappers"; -export class App extends Model< - InferAttributes, - InferCreationAttributes -> { +export class App extends SoftDeletableModel { declare id: CreationOptional; declare createdAt: CreationOptional; declare updatedAt: CreationOptional; @@ -47,6 +45,9 @@ App.init( allowNull: false, defaultValue: DataTypes.NOW, }, + deletedAt: { + type: DataTypes.DATE, + }, updatedAt: { type: DataTypes.DATE, allowNull: false, diff --git a/front/lib/resources/storage/models/data_source.ts b/front/lib/resources/storage/models/data_source.ts index d0f6c72c5af0..462c3e7f9036 100644 --- a/front/lib/resources/storage/models/data_source.ts +++ b/front/lib/resources/storage/models/data_source.ts @@ -1,22 +1,14 @@ import type { ConnectorProvider } from "@dust-tt/types"; -import type { - CreationOptional, - ForeignKey, - InferAttributes, - InferCreationAttributes, - NonAttribute, -} from "sequelize"; -import { DataTypes, Model } from "sequelize"; +import type { CreationOptional, ForeignKey, NonAttribute } from "sequelize"; +import { DataTypes } from "sequelize"; import { User } from "@app/lib/models/user"; import { Workspace } from "@app/lib/models/workspace"; import { frontSequelize } from "@app/lib/resources/storage"; import { VaultModel } from "@app/lib/resources/storage/models/vaults"; +import { SoftDeletableModel } from "@app/lib/resources/storage/wrappers"; -export class DataSourceModel extends Model< - InferAttributes, - InferCreationAttributes -> { +export class DataSourceModel extends SoftDeletableModel { declare id: CreationOptional; declare createdAt: CreationOptional; declare updatedAt: CreationOptional; @@ -52,6 +44,9 @@ DataSourceModel.init( allowNull: false, defaultValue: DataTypes.NOW, }, + deletedAt: { + type: DataTypes.DATE, + }, updatedAt: { type: DataTypes.DATE, allowNull: false, @@ -92,7 +87,7 @@ DataSourceModel.init( modelName: "data_source", sequelize: frontSequelize, indexes: [ - { fields: ["workspaceId", "name"], unique: true }, + { fields: ["workspaceId", "name", "deletedAt"], unique: true }, { fields: ["workspaceId", "connectorProvider"] }, { fields: ["workspaceId", "vaultId"] }, { fields: ["dustAPIProjectId"] }, diff --git a/front/lib/resources/storage/models/data_source_view.ts b/front/lib/resources/storage/models/data_source_view.ts index 9557cf1f7850..f660119d0803 100644 --- a/front/lib/resources/storage/models/data_source_view.ts +++ b/front/lib/resources/storage/models/data_source_view.ts @@ -1,23 +1,15 @@ import type { DataSourceViewKind } from "@dust-tt/types"; -import type { - CreationOptional, - ForeignKey, - InferAttributes, - InferCreationAttributes, - NonAttribute, -} from "sequelize"; -import { DataTypes, Model } from "sequelize"; +import type { CreationOptional, ForeignKey, NonAttribute } from "sequelize"; +import { DataTypes } from "sequelize"; import { User } from "@app/lib/models/user"; import { Workspace } from "@app/lib/models/workspace"; import { frontSequelize } from "@app/lib/resources/storage"; import { DataSourceModel } from "@app/lib/resources/storage/models/data_source"; import { VaultModel } from "@app/lib/resources/storage/models/vaults"; +import { SoftDeletableModel } from "@app/lib/resources/storage/wrappers"; -export class DataSourceViewModel extends Model< - InferAttributes, - InferCreationAttributes -> { +export class DataSourceViewModel extends SoftDeletableModel { declare id: CreationOptional; declare createdAt: CreationOptional; declare updatedAt: CreationOptional; @@ -50,6 +42,9 @@ DataSourceViewModel.init( allowNull: false, defaultValue: DataTypes.NOW, }, + deletedAt: { + type: DataTypes.DATE, + }, editedAt: { type: DataTypes.DATE, allowNull: false, @@ -75,7 +70,10 @@ DataSourceViewModel.init( indexes: [ { fields: ["workspaceId", "id"] }, { fields: ["workspaceId", "vaultId"] }, - { fields: ["workspaceId", "dataSourceId", "vaultId"], unique: true }, + { + fields: ["workspaceId", "dataSourceId", "vaultId", "deletedAt"], + unique: true, + }, ], } ); diff --git a/front/lib/resources/storage/wrappers.ts b/front/lib/resources/storage/wrappers.ts new file mode 100644 index 000000000000..b4cdb8c151a5 --- /dev/null +++ b/front/lib/resources/storage/wrappers.ts @@ -0,0 +1,154 @@ +import type { + Attributes, + CountWithOptions, + CreationOptional, + DestroyOptions, + FindOptions, + GroupedCountResultItem, + InferAttributes, + InferCreationAttributes, + ModelStatic, + UpdateOptions, + WhereOptions, +} from "sequelize"; +import { Model, Op } from "sequelize"; + +export type ModelStaticSoftDeletable = + ModelStatic & { + findAll( + options: WithIncludeDeleted>> + ): Promise; + }; + +type WithHardDelete = T & { + hardDelete: boolean; +}; + +type WithIncludeDeleted = T & { + includeDeleted?: boolean; +}; + +/** + * The `SoftDeletableModel` class extends Sequelize's `Model` to implement custom soft delete + * functionality. This class overrides certain static methods to provide a mechanism for marking + * records as deleted without physically removing them from the database. The `deletedAt` field is + * used to track when a record is soft deleted. + * + * Key Features: + * - **Soft Delete:** The `destroy` method is overridden to perform a soft delete by default, setting + * the `deletedAt` field to the current date and time. A hard delete can be triggered by passing the + * `hardDelete` option. + * - **Filtering Deleted Records:** The `findAll`, `findOne`, and `count` methods are overridden to + * exclude soft-deleted records by default. The `includeDeleted` option can be used to include these + * records in queries. + * - **No Instance Method Override Needed:** Instance methods are not overridden as Sequelize utilizes + * the static methods internally, making this implementation efficient and seamless for + * instance-specific operations. + * + * Usage: + * Extend this class for models that require soft delete functionality. The `deletedAt` field + * is automatically declared and managed by this class. + */ +export class SoftDeletableModel extends Model< + InferAttributes, + InferCreationAttributes +> { + declare deletedAt: CreationOptional; + + // Delete. + + private static async softDelete( + options: DestroyOptions> + ): Promise { + const updateOptions: UpdateOptions> = { + ...options, + fields: ["deletedAt"], + where: options?.where || {}, + }; + + const [affectedCount] = await this.update( + { deletedAt: new Date() }, + updateOptions + ); + + return affectedCount; + } + + public static override async destroy( + options: WithHardDelete>> + ): Promise { + if (options.hardDelete) { + return super.destroy(options); + } + + return this.softDelete(options); + } + + // Fetch. + + public static override async findAll( + this: ModelStatic, + options?: WithIncludeDeleted>> + ): Promise { + if (options?.includeDeleted) { + return super.findAll(options) as Promise; + } + + const whereClause = { + ...options?.where, + deletedAt: { + [Op.is]: null, + }, + } as WhereOptions>; + + return super.findAll({ + ...options, + where: whereClause, + }) as Promise; + } + + public static override async findOne( + this: ModelStatic, + options?: WithIncludeDeleted>> + ): Promise { + if (options?.includeDeleted) { + return super.findOne(options) as Promise; + } + + const whereClause = { + ...options?.where, + deletedAt: { + [Op.is]: null, + }, + } as WhereOptions>; + + return super.findOne({ + ...options, + where: whereClause, + }) as Promise; + } + + public static override count(options?: CountWithOptions): Promise; + public static override count( + options: CountWithOptions & { group: unknown } + ): Promise; + public static override async count( + options?: WithIncludeDeleted + ): Promise { + if (options?.includeDeleted) { + return super.count(options); + } + + const whereClause: WhereOptions = { + ...options?.where, + deletedAt: { + [Op.is]: null, + }, + }; + + return super.count({ + ...options, + where: whereClause, + }); + } +} diff --git a/front/lib/resources/template_resource.ts b/front/lib/resources/template_resource.ts index 88cc34304322..c3169e02d3a4 100644 --- a/front/lib/resources/template_resource.ts +++ b/front/lib/resources/template_resource.ts @@ -79,7 +79,7 @@ export class TemplateResource extends BaseResource { async delete( auth: Authenticator, - transaction?: Transaction + { transaction }: { transaction?: Transaction } = {} ): Promise> { try { await this.model.destroy({ diff --git a/front/lib/resources/types.ts b/front/lib/resources/types.ts index 1bb18923f5e7..542c8b420752 100644 --- a/front/lib/resources/types.ts +++ b/front/lib/resources/types.ts @@ -6,6 +6,8 @@ import type { WhereOptions, } from "sequelize"; +import type { SoftDeletableModel } from "@app/lib/resources/storage/wrappers"; + export type NonAttributeKeys = { [K in keyof M]: M[K] extends NonAttribute> ? K : never; }[keyof M] & @@ -26,9 +28,11 @@ export type TypedIncludeable = { }; }[NonAttributeKeys]; -export interface ResourceFindOptions { +export type ResourceFindOptions = { includes?: TypedIncludeable[]; limit?: number; order?: FindOptions["order"]; where?: WhereOptions; -} +} & (M extends SoftDeletableModel + ? { includeDeleted?: boolean } + : { includeDeleted?: never }); diff --git a/front/lib/resources/user_resource.ts b/front/lib/resources/user_resource.ts index 6e6c64b82487..19bb719a5cbc 100644 --- a/front/lib/resources/user_resource.ts +++ b/front/lib/resources/user_resource.ts @@ -242,7 +242,7 @@ export class UserResource extends BaseResource { async delete( auth: Authenticator, - transaction?: Transaction + { transaction }: { transaction?: Transaction } ): Promise> { try { await this.model.destroy({ diff --git a/front/lib/resources/vault_resource.ts b/front/lib/resources/vault_resource.ts index 407465879e16..c9d8df607519 100644 --- a/front/lib/resources/vault_resource.ts +++ b/front/lib/resources/vault_resource.ts @@ -236,7 +236,7 @@ export class VaultResource extends BaseResource { async delete( auth: Authenticator, - transaction?: Transaction + { transaction }: { transaction?: Transaction } ): Promise> { await GroupVaultModel.destroy({ where: { diff --git a/front/migrations/20231107_empty_data_sources.ts b/front/migrations/20231107_empty_data_sources.ts index b5e0f60afc99..912203d6058d 100644 --- a/front/migrations/20231107_empty_data_sources.ts +++ b/front/migrations/20231107_empty_data_sources.ts @@ -1,115 +1,115 @@ -import { CoreAPI } from "@dust-tt/types"; -import { Sequelize } from "sequelize"; - -import config from "@app/lib/api/config"; -import { Workspace } from "@app/lib/models/workspace"; -import { DataSourceModel } from "@app/lib/resources/storage/models/data_source"; -import logger from "@app/logger/logger"; -import { launchScrubDataSourceWorkflow } from "@app/poke/temporal/client"; - -const { CORE_DATABASE_URI, LIVE } = process.env; - -async function main() { - if (!CORE_DATABASE_URI) { - throw new Error("CORE_DATABASE_URI is not defined"); - } - - const coreSequelize = new Sequelize(CORE_DATABASE_URI, { logging: false }); - - const dataSources = await DataSourceModel.findAll({}); - console.log(`Processing ${dataSources.length} data sources.`); - - let countDeleted = 0; - - for (const ds of dataSources) { - const dustAPIProjectId = ds.dustAPIProjectId; - - /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ - const [dsData, dsMetaData] = (await coreSequelize.query(` - SELECT * FROM data_sources WHERE project = ${dustAPIProjectId}; - `)) as [any[], { rowCount?: number }]; - - if (dsData.length == 0) { - console.log(`[!] CORE Data Source Not Found: ${dustAPIProjectId}`); - continue; - } - if (dsData.length > 1) { - console.log(`[!] CORE Data Source Found >1: ${dustAPIProjectId}`); - continue; - } - - /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ - const [docData, docMetaData] = (await coreSequelize.query(` - SELECT * FROM data_sources_documents WHERE data_source = ${dsData[0].id} AND status='latest' LIMIT 1; - `)) as [any[], { rowCount?: number }]; - - const is2DayOld = - ds.createdAt < new Date(Date.now() - 2 * 24 * 60 * 60 * 1000); - - /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ - const [databaseData, databaseMetaData] = (await coreSequelize.query(` - SELECT * FROM tables WHERE data_source = ${dsData[0].id} LIMIT 1; - `)) as [any[], { rowCount?: number }]; - - if ( - docData.length === 0 && - databaseData.length === 0 && - !ds.connectorId && - is2DayOld - ) { - countDeleted += 1; - console.log( - `[DELETE] Data Source: ${dustAPIProjectId} ${ds.id} ${ds.name} ${dsData[0].internal_id}` - ); - if (LIVE) { - const coreAPI = new CoreAPI(config.getCoreAPIConfig(), logger); - const coreDeleteRes = await coreAPI.deleteDataSource({ - projectId: dustAPIProjectId, - dataSourceId: ds.dustAPIDataSourceId, - }); - if (coreDeleteRes.isErr()) { - console.log("[x] Error deleting CoreAPI data source", ds); - throw new Error( - `Error deleting core data source: ${coreDeleteRes.error.message}` - ); - } - - console.log("[i] Data Source destroyed"); - await ds.destroy(); - - const workspace = await Workspace.findOne({ - where: { - id: ds.workspaceId, - }, - }); - - if (!workspace) { - throw new Error(`Workspace not found: ${ds.workspaceId}`); - } - - console.log( - "Launching scrub workflow", - workspace.sId, - dustAPIProjectId - ); - - await launchScrubDataSourceWorkflow({ - wId: workspace.sId, - dustAPIProjectId, - }); - } - } - } - - console.log(`Deleted ${countDeleted} data sources.`); -} - -main() - .then(() => { - console.log("Done"); - process.exit(0); - }) - .catch((err) => { - console.error(err); - process.exit(1); - }); +// import { CoreAPI } from "@dust-tt/types"; +// import { Sequelize } from "sequelize"; + +// import config from "@app/lib/api/config"; +// import { Workspace } from "@app/lib/models/workspace"; +// import { DataSourceModel } from "@app/lib/resources/storage/models/data_source"; +// import logger from "@app/logger/logger"; +// import { launchScrubDataSourceWorkflow } from "@app/poke/temporal/client"; + +// const { CORE_DATABASE_URI, LIVE } = process.env; + +// async function main() { +// if (!CORE_DATABASE_URI) { +// throw new Error("CORE_DATABASE_URI is not defined"); +// } + +// const coreSequelize = new Sequelize(CORE_DATABASE_URI, { logging: false }); + +// const dataSources = await DataSourceModel.findAll({}); +// console.log(`Processing ${dataSources.length} data sources.`); + +// let countDeleted = 0; + +// for (const ds of dataSources) { +// const dustAPIProjectId = ds.dustAPIProjectId; + +// /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ +// const [dsData, dsMetaData] = (await coreSequelize.query(` +// SELECT * FROM data_sources WHERE project = ${dustAPIProjectId}; +// `)) as [any[], { rowCount?: number }]; + +// if (dsData.length == 0) { +// console.log(`[!] CORE Data Source Not Found: ${dustAPIProjectId}`); +// continue; +// } +// if (dsData.length > 1) { +// console.log(`[!] CORE Data Source Found >1: ${dustAPIProjectId}`); +// continue; +// } + +// /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ +// const [docData, docMetaData] = (await coreSequelize.query(` +// SELECT * FROM data_sources_documents WHERE data_source = ${dsData[0].id} AND status='latest' LIMIT 1; +// `)) as [any[], { rowCount?: number }]; + +// const is2DayOld = +// ds.createdAt < new Date(Date.now() - 2 * 24 * 60 * 60 * 1000); + +// /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ +// const [databaseData, databaseMetaData] = (await coreSequelize.query(` +// SELECT * FROM tables WHERE data_source = ${dsData[0].id} LIMIT 1; +// `)) as [any[], { rowCount?: number }]; + +// if ( +// docData.length === 0 && +// databaseData.length === 0 && +// !ds.connectorId && +// is2DayOld +// ) { +// countDeleted += 1; +// console.log( +// `[DELETE] Data Source: ${dustAPIProjectId} ${ds.id} ${ds.name} ${dsData[0].internal_id}` +// ); +// if (LIVE) { +// const coreAPI = new CoreAPI(config.getCoreAPIConfig(), logger); +// const coreDeleteRes = await coreAPI.deleteDataSource({ +// projectId: dustAPIProjectId, +// dataSourceId: ds.dustAPIDataSourceId, +// }); +// if (coreDeleteRes.isErr()) { +// console.log("[x] Error deleting CoreAPI data source", ds); +// throw new Error( +// `Error deleting core data source: ${coreDeleteRes.error.message}` +// ); +// } + +// console.log("[i] Data Source destroyed"); +// await ds.destroy(); + +// const workspace = await Workspace.findOne({ +// where: { +// id: ds.workspaceId, +// }, +// }); + +// if (!workspace) { +// throw new Error(`Workspace not found: ${ds.workspaceId}`); +// } + +// console.log( +// "Launching scrub workflow", +// workspace.sId, +// dustAPIProjectId +// ); + +// await launchScrubDataSourceWorkflow({ +// wId: workspace.sId, +// dustAPIProjectId, +// }); +// } +// } +// } + +// console.log(`Deleted ${countDeleted} data sources.`); +// } + +// main() +// .then(() => { +// console.log("Done"); +// process.exit(0); +// }) +// .catch((err) => { +// console.error(err); +// process.exit(1); +// }); diff --git a/front/migrations/20240524_clean_up_orphaned_core_data_sources.ts b/front/migrations/20240524_clean_up_orphaned_core_data_sources.ts index d327bc704efc..b26b47d823dc 100644 --- a/front/migrations/20240524_clean_up_orphaned_core_data_sources.ts +++ b/front/migrations/20240524_clean_up_orphaned_core_data_sources.ts @@ -1,30 +1,30 @@ -import { CoreAPI } from "@dust-tt/types"; +// import { CoreAPI } from "@dust-tt/types"; -import config from "@app/lib/api/config"; -import logger from "@app/logger/logger"; -import { launchScrubDataSourceWorkflow } from "@app/poke/temporal/client"; -import { makeScript } from "@app/scripts/helpers"; +// import config from "@app/lib/api/config"; +// import logger from "@app/logger/logger"; +// import { launchScrubDataSourceWorkflow } from "@app/poke/temporal/client"; +// import { makeScript } from "@app/scripts/helpers"; -const ORPHANED_DATA_SOURCES: { project: string; data_source_id: string }[] = [ - // add orphaned data sources here -]; +// const ORPHANED_DATA_SOURCES: { project: string; data_source_id: string }[] = [ +// // add orphaned data sources here +// ]; -makeScript({}, async ({ execute }) => { - if (execute) { - for (const { project, data_source_id } of ORPHANED_DATA_SOURCES) { - const coreAPI = new CoreAPI(config.getCoreAPIConfig(), logger); - const coreDeleteRes = await coreAPI.deleteDataSource({ - projectId: project, - dataSourceId: data_source_id, - }); - if (coreDeleteRes.isErr()) { - console.log("ERROR:" + coreDeleteRes.error); - } +// makeScript({}, async ({ execute }) => { +// if (execute) { +// for (const { project, data_source_id } of ORPHANED_DATA_SOURCES) { +// const coreAPI = new CoreAPI(config.getCoreAPIConfig(), logger); +// const coreDeleteRes = await coreAPI.deleteDataSource({ +// projectId: project, +// dataSourceId: data_source_id, +// }); +// if (coreDeleteRes.isErr()) { +// console.log("ERROR:" + coreDeleteRes.error); +// } - await launchScrubDataSourceWorkflow({ - wId: "scrub_orphaned", - dustAPIProjectId: project, - }); - } - } -}); +// await launchScrubDataSourceWorkflow({ +// wId: "scrub_orphaned", +// dustAPIProjectId: project, +// }); +// } +// } +// }); diff --git a/front/migrations/20240730_delete_unmanaged_data_source_views.ts b/front/migrations/20240730_delete_unmanaged_data_source_views.ts index e44dfe93e8c7..1cd956c72cba 100644 --- a/front/migrations/20240730_delete_unmanaged_data_source_views.ts +++ b/front/migrations/20240730_delete_unmanaged_data_source_views.ts @@ -45,7 +45,7 @@ async function deleteUnmanagedDataSourceViewsForWorkspace( } for (const view of viewsToDelete) { - await view.delete(auth); + await view.delete(auth, { hardDelete: true }); logger.info(`Deleted view for data source ${view.dataSourceId}.`); } diff --git a/front/migrations/20240927_clean_up_orphaned_core_data_sources.ts b/front/migrations/20240927_clean_up_orphaned_core_data_sources.ts index 2c2e6fdae54e..5195f2ad210c 100644 --- a/front/migrations/20240927_clean_up_orphaned_core_data_sources.ts +++ b/front/migrations/20240927_clean_up_orphaned_core_data_sources.ts @@ -1,87 +1,87 @@ -import { CoreAPI } from "@dust-tt/types"; -import { Sequelize } from "sequelize"; +// import { CoreAPI } from "@dust-tt/types"; +// import { Sequelize } from "sequelize"; -import config from "@app/lib/api/config"; -import { DataSourceResource } from "@app/lib/resources/data_source_resource"; -import logger from "@app/logger/logger"; -import { launchScrubDataSourceWorkflow } from "@app/poke/temporal/client"; -import { makeScript } from "@app/scripts/helpers"; +// import config from "@app/lib/api/config"; +// import { DataSourceResource } from "@app/lib/resources/data_source_resource"; +// import logger from "@app/logger/logger"; +// import { launchScrubDataSourceWorkflow } from "@app/poke/temporal/client"; +// import { makeScript } from "@app/scripts/helpers"; -const { CORE_DATABASE_URI } = process.env; +// const { CORE_DATABASE_URI } = process.env; -makeScript({}, async ({ execute }) => { - const corePrimary = new Sequelize(CORE_DATABASE_URI as string, { - logging: false, - }); +// makeScript({}, async ({ execute }) => { +// const corePrimary = new Sequelize(CORE_DATABASE_URI as string, { +// logging: false, +// }); - const coreData = await corePrimary.query( - `SELECT id, project, data_source_id, internal_id FROM data_sources` - ); +// const coreData = await corePrimary.query( +// `SELECT id, project, data_source_id, internal_id FROM data_sources` +// ); - const coreDataSources = coreData[0] as { - id: number; - project: number; - data_source_id: string; - internal_id: string; - }[]; +// const coreDataSources = coreData[0] as { +// id: number; +// project: number; +// data_source_id: string; +// internal_id: string; +// }[]; - const frontDataSources = await DataSourceResource.model.findAll({}); +// const frontDataSources = await DataSourceResource.model.findAll({}); - logger.info( - { - coreDataSources: coreDataSources.length, - frontDataSources: frontDataSources.length, - }, - "Retrieved data sources" - ); +// logger.info( +// { +// coreDataSources: coreDataSources.length, +// frontDataSources: frontDataSources.length, +// }, +// "Retrieved data sources" +// ); - const frontDataSourcesById = frontDataSources.reduce( - (acc, ds) => { - acc[`${ds.dustAPIProjectId}-${ds.dustAPIDataSourceId}`] = ds; - return acc; - }, - {} as Record - ); +// const frontDataSourcesById = frontDataSources.reduce( +// (acc, ds) => { +// acc[`${ds.dustAPIProjectId}-${ds.dustAPIDataSourceId}`] = ds; +// return acc; +// }, +// {} as Record +// ); - for (const coreDataSource of coreDataSources) { - if ( - !frontDataSourcesById[ - `${coreDataSource.project}-${coreDataSource.data_source_id}` - ] - ) { - const coreData: any = await corePrimary.query( - `SELECT COUNT(*) FROM data_sources_documents WHERE data_source=${coreDataSource.id}` - ); +// for (const coreDataSource of coreDataSources) { +// if ( +// !frontDataSourcesById[ +// `${coreDataSource.project}-${coreDataSource.data_source_id}` +// ] +// ) { +// const coreData: any = await corePrimary.query( +// `SELECT COUNT(*) FROM data_sources_documents WHERE data_source=${coreDataSource.id}` +// ); - logger.info( - { - coreDataSource, - coreDocuments: coreData[0][0]["count"], - }, - "Found orphaned core data source" - ); +// logger.info( +// { +// coreDataSource, +// coreDocuments: coreData[0][0]["count"], +// }, +// "Found orphaned core data source" +// ); - if (execute) { - const coreAPI = new CoreAPI(config.getCoreAPIConfig(), logger); - const coreDeleteRes = await coreAPI.deleteDataSource({ - projectId: `${coreDataSource.project}`, - dataSourceId: coreDataSource.data_source_id, - }); - if (coreDeleteRes.isErr()) { - logger.error( - { - coreDeleteRes, - }, - "Failed to delete core data source" - ); - return; - } +// if (execute) { +// const coreAPI = new CoreAPI(config.getCoreAPIConfig(), logger); +// const coreDeleteRes = await coreAPI.deleteDataSource({ +// projectId: `${coreDataSource.project}`, +// dataSourceId: coreDataSource.data_source_id, +// }); +// if (coreDeleteRes.isErr()) { +// logger.error( +// { +// coreDeleteRes, +// }, +// "Failed to delete core data source" +// ); +// return; +// } - await launchScrubDataSourceWorkflow({ - wId: "scrub_orphaned", - dustAPIProjectId: `${coreDataSource.project}`, - }); - } - } - } -}); +// await launchScrubDataSourceWorkflow({ +// wId: "scrub_orphaned", +// dustAPIProjectId: `${coreDataSource.project}`, +// }); +// } +// } +// } +// }); diff --git a/front/migrations/db/migration_97.sql b/front/migrations/db/migration_97.sql new file mode 100644 index 000000000000..d0564378a82a --- /dev/null +++ b/front/migrations/db/migration_97.sql @@ -0,0 +1,24 @@ +-- Migration created on Sep 25, 2024 +ALTER TABLE "public"."data_sources" +ADD COLUMN "deletedAt" TIMESTAMP +WITH + TIME ZONE; + +CREATE UNIQUE INDEX CONCURRENTLY "data_sources_workspace_id_name_deleted_at" ON "data_sources" ("workspaceId", "name", "deletedAt"); + +ALTER TABLE "public"."data_source_views" +ADD COLUMN "deletedAt" TIMESTAMP +WITH + TIME ZONE; + +CREATE UNIQUE INDEX CONCURRENTLY "data_source_views_workspace_id_data_source_id_vault_id_deleted_at" ON "data_source_views" ( + "workspaceId", + "dataSourceId", + "vaultId", + "deletedAt" +); + +ALTER TABLE "public"."apps" +ADD COLUMN "deletedAt" TIMESTAMP +WITH + TIME ZONE; \ No newline at end of file diff --git a/front/pages/api/poke/workspaces/[wId]/data_sources/[dsId]/index.ts b/front/pages/api/poke/workspaces/[wId]/data_sources/[dsId]/index.ts index cb702075c85c..8ea7c57e525b 100644 --- a/front/pages/api/poke/workspaces/[wId]/data_sources/[dsId]/index.ts +++ b/front/pages/api/poke/workspaces/[wId]/data_sources/[dsId]/index.ts @@ -3,7 +3,7 @@ import type { WithAPIErrorResponse } from "@dust-tt/types"; import { assertNever } from "@dust-tt/types"; import type { NextApiRequest, NextApiResponse } from "next"; -import { deleteDataSource } from "@app/lib/api/data_sources"; +import { softDeleteDataSourceAndLaunchScrubWorkflow } from "@app/lib/api/data_sources"; import { withSessionAuthentication } from "@app/lib/api/wrappers"; import { Authenticator } from "@app/lib/auth"; import type { SessionWithUser } from "@app/lib/iam/provider"; @@ -98,7 +98,10 @@ async function handler( }); } - const delRes = await deleteDataSource(auth, dataSource); + const delRes = await softDeleteDataSourceAndLaunchScrubWorkflow( + auth, + dataSource + ); if (delRes.isErr()) { switch (delRes.error.code) { case "unauthorized_deletion": diff --git a/front/pages/api/w/[wId]/data_sources/[dsId]/index.ts b/front/pages/api/w/[wId]/data_sources/[dsId]/index.ts index 46b54acd06f3..24fd6f8dccd0 100644 --- a/front/pages/api/w/[wId]/data_sources/[dsId]/index.ts +++ b/front/pages/api/w/[wId]/data_sources/[dsId]/index.ts @@ -2,7 +2,7 @@ import type { DataSourceType, WithAPIErrorResponse } from "@dust-tt/types"; import { MANAGED_DS_DELETABLE } from "@dust-tt/types"; import type { NextApiRequest, NextApiResponse } from "next"; -import { deleteDataSource } from "@app/lib/api/data_sources"; +import { softDeleteDataSourceAndLaunchScrubWorkflow } from "@app/lib/api/data_sources"; import { withSessionAuthenticationForWorkspace } from "@app/lib/api/wrappers"; import type { Authenticator } from "@app/lib/auth"; import { DataSourceResource } from "@app/lib/resources/data_source_resource"; @@ -138,7 +138,10 @@ async function handler( }); } - const dRes = await deleteDataSource(auth, dataSource); + const dRes = await softDeleteDataSourceAndLaunchScrubWorkflow( + auth, + dataSource + ); if (dRes.isErr()) { return apiError(req, res, { status_code: 500, diff --git a/front/pages/api/w/[wId]/data_sources/managed.ts b/front/pages/api/w/[wId]/data_sources/managed.ts index 5ecf4943fdaa..6a4e9554319b 100644 --- a/front/pages/api/w/[wId]/data_sources/managed.ts +++ b/front/pages/api/w/[wId]/data_sources/managed.ts @@ -337,7 +337,10 @@ async function handler( }, "Failed to create the connector" ); - await dataSource.delete(auth); + + // If the connector creation fails, we delete the data source and the project. + await dataSource.delete(auth, { hardDelete: true }); + const deleteRes = await coreAPI.deleteDataSource({ projectId: dustProject.value.project.project_id.toString(), dataSourceId: dustDataSource.value.data_source.data_source_id, diff --git a/front/pages/api/w/[wId]/vaults/[vId]/data_source_views/[dsvId]/index.ts b/front/pages/api/w/[wId]/vaults/[vId]/data_source_views/[dsvId]/index.ts index 2fe85928a26b..11588aec4a2c 100644 --- a/front/pages/api/w/[wId]/vaults/[vId]/data_source_views/[dsvId]/index.ts +++ b/front/pages/api/w/[wId]/vaults/[vId]/data_source_views/[dsvId]/index.ts @@ -114,17 +114,8 @@ async function handler( }); } - const deleteResult = await dataSourceView.delete(auth); - - if (deleteResult.isErr()) { - return apiError(req, res, { - status_code: 500, - api_error: { - type: "internal_server_error", - message: "The data source view cannot be updated.", - }, - }); - } + // Directly, hard delete the data source view. + await dataSourceView.delete(auth, { hardDelete: true }); res.status(204).end(); return; diff --git a/front/pages/api/w/[wId]/vaults/[vId]/data_sources/[dsId]/index.ts b/front/pages/api/w/[wId]/vaults/[vId]/data_sources/[dsId]/index.ts index d40be8f59a73..6babe2772c21 100644 --- a/front/pages/api/w/[wId]/vaults/[vId]/data_sources/[dsId]/index.ts +++ b/front/pages/api/w/[wId]/vaults/[vId]/data_sources/[dsId]/index.ts @@ -5,7 +5,7 @@ import * as t from "io-ts"; import * as reporter from "io-ts-reporters"; import type { NextApiRequest, NextApiResponse } from "next"; -import { deleteDataSource } from "@app/lib/api/data_sources"; +import { softDeleteDataSourceAndLaunchScrubWorkflow } from "@app/lib/api/data_sources"; import { withSessionAuthenticationForWorkspace } from "@app/lib/api/wrappers"; import type { Authenticator } from "@app/lib/auth"; import { DataSourceResource } from "@app/lib/resources/data_source_resource"; @@ -156,7 +156,10 @@ async function handler( }); } - const dRes = await deleteDataSource(auth, dataSource); + const dRes = await softDeleteDataSourceAndLaunchScrubWorkflow( + auth, + dataSource + ); if (dRes.isErr()) { return apiError(req, res, { status_code: 500, diff --git a/front/pages/api/w/[wId]/vaults/[vId]/data_sources/index.ts b/front/pages/api/w/[wId]/vaults/[vId]/data_sources/index.ts index dca4fc6e3239..da5d0e32b9e7 100644 --- a/front/pages/api/w/[wId]/vaults/[vId]/data_sources/index.ts +++ b/front/pages/api/w/[wId]/vaults/[vId]/data_sources/index.ts @@ -419,7 +419,10 @@ const handleDataSourceWithProvider = async ({ }, "Failed to create the connector" ); - await dataSource.delete(auth); + + // Rollback the data source creation. + await dataSource.delete(auth, { hardDelete: true }); + const deleteRes = await coreAPI.deleteDataSource({ projectId: dustProject.value.project.project_id.toString(), dataSourceId: dustDataSource.value.data_source.data_source_id, diff --git a/front/pages/api/w/[wId]/vaults/[vId]/index.ts b/front/pages/api/w/[wId]/vaults/[vId]/index.ts index c1d68abf8f74..788992f6493a 100644 --- a/front/pages/api/w/[wId]/vaults/[vId]/index.ts +++ b/front/pages/api/w/[wId]/vaults/[vId]/index.ts @@ -189,7 +189,9 @@ async function handler( for (const dataSourceId of Object.keys(viewByDataSourceId)) { if (!content.map((c) => c.dataSourceId).includes(dataSourceId)) { const view = viewByDataSourceId[dataSourceId]; - await view.delete(auth); + + // Hard delete previous views. + await view.delete(auth, { hardDelete: true }); } } } diff --git a/front/poke/temporal/activities.ts b/front/poke/temporal/activities.ts index 3f1fe126e5fb..0d9c00397ab0 100644 --- a/front/poke/temporal/activities.ts +++ b/front/poke/temporal/activities.ts @@ -4,6 +4,7 @@ import { chunk } from "lodash"; import { Op } from "sequelize"; import config from "@app/lib/api/config"; +import { hardDeleteDataSource } from "@app/lib/api/data_sources"; import { Authenticator } from "@app/lib/auth"; import { AgentBrowseAction } from "@app/lib/models/assistant/actions/browse"; import { AgentDataSourceConfiguration } from "@app/lib/models/assistant/actions/data_sources"; @@ -59,9 +60,11 @@ import logger from "@app/logger/logger"; const { DUST_DATA_SOURCES_BUCKET, SERVICE_ACCOUNT } = process.env; export async function scrubDataSourceActivity({ - dustAPIProjectId, + dataSourceId, + workspaceId, }: { - dustAPIProjectId: string; + dataSourceId: string; + workspaceId: string; }) { if (!SERVICE_ACCOUNT) { throw new Error("SERVICE_ACCOUNT is not set."); @@ -70,6 +73,30 @@ export async function scrubDataSourceActivity({ throw new Error("DUST_DATA_SOURCES_BUCKET is not set."); } + const auth = await Authenticator.internalAdminForWorkspace(workspaceId); + const dataSource = await DataSourceResource.fetchById(auth, dataSourceId, { + includeDeleted: true, + }); + if (!dataSource) { + logger.info( + { dataSource: { sId: dataSourceId } }, + "Data source not found." + ); + + throw new Error("Data source not found."); + } + + // Ensure the data source has been soft deleted. + if (!dataSource.deletedAt) { + logger.info( + { dataSource: { sId: dataSourceId } }, + "Data source is not soft deleted." + ); + throw new Error("Data source is not soft deleted."); + } + + const { dustAPIProjectId } = dataSource; + const storage = new Storage({ keyFilename: SERVICE_ACCOUNT }); const [files] = await storage @@ -95,6 +122,8 @@ export async function scrubDataSourceActivity({ }) ); } + + return hardDeleteDataSource(auth, dataSource); } export async function isWorkflowDeletableActivity({ @@ -396,7 +425,7 @@ export async function deleteAppsActivity({ throw new Error(`Error deleting Project from Core: ${res.error.message}`); } - const delRes = await app.delete(auth); + const delRes = await app.delete(auth, { hardDelete: true }); if (delRes.isErr()) { throw new Error(`Error deleting App ${app.sId}: ${delRes.error.message}`); } @@ -505,12 +534,12 @@ export async function deleteMembersActivity({ ); // Delete the user's files await FileResource.deleteAllForUser(user.toJSON(), t); - await membership.delete(auth, t); - await user.delete(auth, t); + await membership.delete(auth, { transaction: t }); + await user.delete(auth, { transaction: t }); } } else { logger.info(`[Workspace delete] Deleting Membership ${membership.id}`); - await membership.delete(auth, t); + await membership.delete(auth, { transaction: t }); } } }); diff --git a/front/poke/temporal/client.ts b/front/poke/temporal/client.ts index be8ed305a1e3..deaed295cd67 100644 --- a/front/poke/temporal/client.ts +++ b/front/poke/temporal/client.ts @@ -1,34 +1,37 @@ +import type { LightWorkspaceType } from "@dust-tt/types"; import { Err } from "@dust-tt/types"; import { WorkflowExecutionAlreadyStartedError } from "@temporalio/client"; +import type { DataSourceResource } from "@app/lib/resources/data_source_resource"; import { getTemporalClient } from "@app/lib/temporal"; import logger from "@app/logger/logger"; import { deleteWorkspaceWorkflow, scrubDataSourceWorkflow } from "./workflows"; -export async function launchScrubDataSourceWorkflow({ - wId, - dustAPIProjectId, -}: { - wId: string; - dustAPIProjectId: string; -}) { +export async function launchScrubDataSourceWorkflow( + owner: LightWorkspaceType, + dataSource: DataSourceResource +) { const client = await getTemporalClient(); + try { await client.workflow.start(scrubDataSourceWorkflow, { args: [ { - dustAPIProjectId, + dataSourceId: dataSource.sId, + workspaceId: owner.sId, }, ], taskQueue: "poke-queue", - workflowId: `poke-${wId}-scrub-data-source-${dustAPIProjectId}`, + workflowId: `poke-${owner.sId}-scrub-data-source-${dataSource.sId}`, }); } catch (e) { if (!(e instanceof WorkflowExecutionAlreadyStartedError)) { logger.error( { - wId, + owner: { + sId: owner.sId, + }, error: e, }, "Failed starting scrub data source workflow." diff --git a/front/poke/temporal/workflows.ts b/front/poke/temporal/workflows.ts index 73a4676dfd22..4fd2cf72410f 100644 --- a/front/poke/temporal/workflows.ts +++ b/front/poke/temporal/workflows.ts @@ -19,11 +19,13 @@ const { } = activityProxies; export async function scrubDataSourceWorkflow({ - dustAPIProjectId, + dataSourceId, + workspaceId, }: { - dustAPIProjectId: string; + dataSourceId: string; + workspaceId: string; }) { - await scrubDataSourceActivity({ dustAPIProjectId }); + await scrubDataSourceActivity({ dataSourceId, workspaceId }); } export async function deleteWorkspaceWorkflow({ diff --git a/front/temporal/scrub_workspace/activities.ts b/front/temporal/scrub_workspace/activities.ts index cfdc2bc27491..6f2d2e16a1d8 100644 --- a/front/temporal/scrub_workspace/activities.ts +++ b/front/temporal/scrub_workspace/activities.ts @@ -8,7 +8,10 @@ import { import { destroyConversation } from "@app/lib/api/assistant/conversation/destroy"; import { isGlobalAgentId } from "@app/lib/api/assistant/global_agents"; import config from "@app/lib/api/config"; -import { deleteDataSource, getDataSources } from "@app/lib/api/data_sources"; +import { + getDataSources, + softDeleteDataSourceAndLaunchScrubWorkflow, +} from "@app/lib/api/data_sources"; import { sendAdminDataDeletionEmail } from "@app/lib/api/email"; import { getMembers, @@ -153,7 +156,8 @@ async function deleteDatasources(auth: Authenticator) { // First, we delete all the data source views. const dataSourceViews = await DataSourceViewResource.listByWorkspace(auth); for (const dataSourceView of dataSourceViews) { - const r = await dataSourceView.delete(auth); + // Soft delete the data source view. + const r = await dataSourceView.delete(auth, { hardDelete: false }); if (r.isErr()) { throw new Error(`Failed to delete data source view: ${r.error.message}`); } @@ -162,7 +166,10 @@ async function deleteDatasources(auth: Authenticator) { // Then, we delete all the data sources. const dataSources = await getDataSources(auth); for (const dataSource of dataSources) { - const r = await deleteDataSource(auth, dataSource); + const r = await softDeleteDataSourceAndLaunchScrubWorkflow( + auth, + dataSource + ); if (r.isErr()) { throw new Error(`Failed to delete data source: ${r.error.message}`); }