diff --git a/libraries/grpc-sdk/src/modules/storage/index.ts b/libraries/grpc-sdk/src/modules/storage/index.ts index f1866fb32..405ba3f8e 100644 --- a/libraries/grpc-sdk/src/modules/storage/index.ts +++ b/libraries/grpc-sdk/src/modules/storage/index.ts @@ -30,7 +30,7 @@ export class Storage extends ConduitModule { } createFile( - name: string, + name: string | undefined, data: string, folder?: string, container?: string, @@ -38,6 +38,7 @@ export class Storage extends ConduitModule { isPublic: boolean = false, userId?: string, scope?: string, + alias?: string, ): Promise { return this.client!.createFile({ name, @@ -48,6 +49,7 @@ export class Storage extends ConduitModule { container, userId, scope, + alias, }); } @@ -60,6 +62,7 @@ export class Storage extends ConduitModule { mimeType?: string, userId?: string, scope?: string, + alias?: string, ): Promise { return this.client!.updateFile({ name, @@ -70,6 +73,7 @@ export class Storage extends ConduitModule { container, userId, scope, + alias, }); } @@ -78,7 +82,7 @@ export class Storage extends ConduitModule { } createFileByUrl( - name: string, + name: string | undefined, folder?: string, container?: string, mimeType?: string, @@ -86,6 +90,7 @@ export class Storage extends ConduitModule { isPublic: boolean = false, userId?: string, scope?: string, + alias?: string, ) { return this.client!.createFileByUrl({ name, @@ -96,6 +101,7 @@ export class Storage extends ConduitModule { isPublic, userId, scope, + alias, }); } @@ -108,6 +114,7 @@ export class Storage extends ConduitModule { size?: number, userId?: string, scope?: string, + alias?: string, ) { return this.client!.updateFileByUrl({ id, @@ -118,6 +125,7 @@ export class Storage extends ConduitModule { size, userId, scope, + alias, }); } } diff --git a/modules/storage/src/admin/adminFile.ts b/modules/storage/src/admin/adminFile.ts index 8363228b4..628b72869 100644 --- a/modules/storage/src/admin/adminFile.ts +++ b/modules/storage/src/admin/adminFile.ts @@ -2,7 +2,6 @@ import { ConduitGrpcSdk, DatabaseProvider, GrpcError, - Indexable, ParsedRouterRequest, UnparsedRouterResponse, } from '@conduitplatform/grpc-sdk'; @@ -11,7 +10,15 @@ import { status } from '@grpc/grpc-js'; import { isNil, isString } from 'lodash-es'; import { _StorageContainer, _StorageFolder, File } from '../models/index.js'; import { IStorageProvider } from '../interfaces/index.js'; -import { deepPathHandler, normalizeFolderPath } from '../utils/index.js'; +import { + _createFileUploadUrl, + _updateFile, + _updateFileUploadUrl, + deepPathHandler, + normalizeFolderPath, + storeNewFile, + validateName, +} from '../utils/index.js'; export class AdminFileHandlers { private readonly database: DatabaseProvider; @@ -46,7 +53,7 @@ export class AdminFileHandlers { } async createFile(call: ParsedRouterRequest): Promise { - const { name, data, container, mimeType, isPublic } = call.request.params; + const { name, alias, data, container, mimeType, isPublic } = call.request.params; const folder = normalizeFolderPath(call.request.params.folder); const config = ConfigController.getInstance().config; const usedContainer = isNil(container) @@ -55,29 +62,21 @@ export class AdminFileHandlers { if (folder !== '/') { await this.findOrCreateFolders(folder, usedContainer, isPublic); } - - const exists = await File.getInstance().findOne({ - name, - container: usedContainer, - folder, - }); - if (exists) { - throw new GrpcError(status.ALREADY_EXISTS, 'File already exists'); - } + const validatedName = await validateName(name, folder, usedContainer); if (!isString(data)) { throw new GrpcError(status.INVALID_ARGUMENT, 'Invalid data provided'); } try { - const file = await this.storeNewFile( + return await storeNewFile(this.storageProvider, { + name: validatedName, + alias, data, - usedContainer, + container: usedContainer, folder, isPublic, - name, mimeType, - ); - return file; + }); } catch (e) { throw new GrpcError( status.INTERNAL, @@ -87,7 +86,7 @@ export class AdminFileHandlers { } async createFileUploadUrl(call: ParsedRouterRequest): Promise { - const { name, container, size = 0, mimeType, isPublic } = call.request.params; + const { name, alias, container, size = 0, mimeType, isPublic } = call.request.params; const folder = normalizeFolderPath(call.request.params.folder); const config = ConfigController.getInstance().config; const usedContainer = isNil(container) @@ -96,26 +95,18 @@ export class AdminFileHandlers { if (folder !== '/') { await this.findOrCreateFolders(folder, usedContainer, isPublic); } - - const exists = await File.getInstance().findOne({ - name, - container: usedContainer, - folder, - }); - if (exists) { - throw new GrpcError(status.ALREADY_EXISTS, 'File already exists'); - } + const validatedName = await validateName(name, folder, usedContainer); try { - const { file, url } = await this._createFileUploadUrl( - usedContainer, + return await _createFileUploadUrl(this.storageProvider, { + container: usedContainer, folder, isPublic, - name, + name: validatedName, + alias, size, mimeType, - ); - return { file, url }; + }); } catch (e) { throw new GrpcError( status.INTERNAL, @@ -125,7 +116,7 @@ export class AdminFileHandlers { } async updateFileUploadUrl(call: ParsedRouterRequest): Promise { - const { id, mimeType, size } = call.request.params; + const { id, alias, mimeType, size } = call.request.params; const found = await File.getInstance().findOne({ _id: id }); if (isNil(found)) { throw new GrpcError(status.NOT_FOUND, 'File does not exist'); @@ -135,14 +126,14 @@ export class AdminFileHandlers { found, ); try { - return await this._updateFileUploadUrl( + return await _updateFileUploadUrl(this.storageProvider, found, { name, + alias, folder, container, - mimeType ?? found.mimeType, - found, + mimeType: mimeType ?? found.mimeType, size, - ); + }); } catch (e) { throw new GrpcError( status.INTERNAL, @@ -152,7 +143,7 @@ export class AdminFileHandlers { } async updateFile(call: ParsedRouterRequest): Promise { - const { id, data, mimeType } = call.request.params; + const { id, alias, data, mimeType } = call.request.params; const found = await File.getInstance().findOne({ _id: id }); if (isNil(found)) { throw new GrpcError(status.NOT_FOUND, 'File does not exist'); @@ -162,14 +153,14 @@ export class AdminFileHandlers { found, ); try { - return await this._updateFile( + return await _updateFile(this.storageProvider, found, { name, + alias, folder, container, - Buffer.from(data, 'base64'), - mimeType ?? found.mimeType, - found, - ); + data: Buffer.from(data, 'base64'), + mimeType: mimeType ?? found.mimeType, + }); } catch (e) { throw new GrpcError( status.INTERNAL, @@ -320,77 +311,6 @@ export class AdminFileHandlers { return createdFolders; } - private async storeNewFile( - data: string, - container: string, - folder: string, - isPublic: boolean, - name: string, - mimeType: string, - ): Promise { - const buffer = Buffer.from(data, 'base64'); - const size = buffer.byteLength; - await this.storageProvider - .container(container) - .store((folder === '/' ? '' : folder) + name, buffer, isPublic); - const publicUrl = isPublic - ? await this.storageProvider - .container(container) - .getPublicUrl((folder === '/' ? '' : folder) + name) - : null; - ConduitGrpcSdk.Metrics?.increment('files_total'); - ConduitGrpcSdk.Metrics?.increment('storage_size_bytes_total', size); - return await File.getInstance().create({ - name, - mimeType, - folder: folder, - container: container, - size, - isPublic, - url: publicUrl, - }); - } - - private async _createFileUploadUrl( - container: string, - folder: string, - isPublic: boolean, - name: string, - size: number, - mimeType: string, - ): Promise<{ file: File; url: string }> { - await this.storageProvider - .container(container) - .store( - (folder === '/' ? '' : folder) + name, - Buffer.from('PENDING UPLOAD'), - isPublic, - ); - const publicUrl = isPublic - ? await this.storageProvider - .container(container) - .getPublicUrl((folder === '/' ? '' : folder) + name) - : null; - ConduitGrpcSdk.Metrics?.increment('files_total'); - ConduitGrpcSdk.Metrics?.increment('storage_size_bytes_total', size); - const file = await File.getInstance().create({ - name, - mimeType, - size, - folder: folder, - container: container, - isPublic, - url: publicUrl, - }); - const url = (await this.storageProvider - .container(container) - .getUploadUrl((folder === '/' ? '' : folder) + name)) as string; - return { - file, - url, - }; - } - private async validateFilenameAndContainer(call: ParsedRouterRequest, file: File) { const { name, folder, container } = call.request.params; const newName = name ?? file.name; @@ -420,93 +340,4 @@ export class AdminFileHandlers { container: newContainer, }; } - - private async _updateFileUploadUrl( - name: string, - folder: string, - container: string, - mimeType: string, - file: File, - size: number | undefined | null, - ): Promise<{ file: File; url: string }> { - let updatedFile; - const onlyDataUpdate = - name === file.name && folder === file.folder && container === file.container; - if (onlyDataUpdate) { - updatedFile = await File.getInstance().findByIdAndUpdate(file._id, { - mimeType, - ...{ size: size ?? file.size }, - }); - } else { - await this.storageProvider - .container(container) - .store( - (folder === '/' ? '' : folder) + name, - Buffer.from('PENDING UPLOAD'), - file.isPublic, - ); - await this.storageProvider - .container(file.container) - .delete((file.folder === '/' ? '' : file.folder) + file.name); - const url = file.isPublic - ? await this.storageProvider - .container(container) - .getPublicUrl((folder === '/' ? '' : folder) + name) - : null; - updatedFile = await File.getInstance().findByIdAndUpdate(file._id, { - name, - folder, - container, - url, - mimeType, - ...{ size: size ?? file.size }, - }); - } - if (!isNil(size)) this.updateFileMetrics(file.size, size!); - const uploadUrl = (await this.storageProvider - .container(container) - .getUploadUrl((folder === '/' ? '' : folder) + name)) as string; - return { file: updatedFile!, url: uploadUrl }; - } - - private async _updateFile( - name: string, - folder: string, - container: string, - data: Buffer, - mimeType: string, - file: File, - ): Promise { - const onlyDataUpdate = - name === file.name && folder === file.folder && container === file.container; - await this.storageProvider - .container(container) - .store((folder === '/' ? '' : folder) + name, data, file.isPublic); - if (!onlyDataUpdate) { - await this.storageProvider - .container(file.container) - .delete((file.folder === '/' ? '' : file.folder) + file.name); - } - const url = file.isPublic - ? await this.storageProvider - .container(container) - .getPublicUrl((folder === '/' ? '' : folder) + name) - : null; - const updatedFile = (await File.getInstance().findByIdAndUpdate(file._id, { - name, - folder, - container, - url, - mimeType, - })) as File; - this.updateFileMetrics(file.size, data.byteLength); - return updatedFile; - } - - private updateFileMetrics(currentSize: number, newSize: number) { - const fileSizeDiff = Math.abs(currentSize - newSize); - fileSizeDiff < 0 - ? ConduitGrpcSdk.Metrics?.increment('storage_size_bytes_total', fileSizeDiff) - : ConduitGrpcSdk.Metrics?.decrement('storage_size_bytes_total', fileSizeDiff); - } } diff --git a/modules/storage/src/admin/index.ts b/modules/storage/src/admin/index.ts index ba1c18008..2e1750eee 100644 --- a/modules/storage/src/admin/index.ts +++ b/modules/storage/src/admin/index.ts @@ -214,7 +214,8 @@ export class AdminRoutes { action: ConduitRouteActions.POST, description: `Creates a new file.`, bodyParams: { - name: ConduitString.Required, + name: ConduitString.Optional, + alias: ConduitString.Optional, data: ConduitString.Required, folder: ConduitString.Optional, container: ConduitString.Optional, @@ -228,7 +229,8 @@ export class AdminRoutes { this.routingManager.route( { bodyParams: { - name: { type: TYPE.String, required: true }, + name: { type: TYPE.String, required: false }, + alias: { type: TYPE.String, required: false }, mimeType: TYPE.String, folder: { type: TYPE.String, required: false }, size: { type: TYPE.Number, required: false }, @@ -255,6 +257,7 @@ export class AdminRoutes { }, bodyParams: { name: ConduitString.Optional, + alias: ConduitString.Optional, folder: ConduitString.Optional, container: ConduitString.Optional, data: ConduitString.Required, @@ -271,6 +274,7 @@ export class AdminRoutes { }, bodyParams: { name: ConduitString.Optional, + alias: ConduitString.Optional, folder: ConduitString.Optional, container: ConduitString.Optional, mimeType: ConduitString.Optional, diff --git a/modules/storage/src/config/config.ts b/modules/storage/src/config/config.ts index 05e06bf2c..222a806e3 100644 --- a/modules/storage/src/config/config.ts +++ b/modules/storage/src/config/config.ts @@ -72,4 +72,9 @@ export default { default: '/var/tmp', }, }, + suffixOnNameConflict: { + format: 'Boolean', + doc: 'Defines if a suffix should be appended to the name of a file, upon creation, when name already exists', + default: false, + }, }; diff --git a/modules/storage/src/handlers/file.ts b/modules/storage/src/handlers/file.ts index be90546bc..62ed02ab6 100644 --- a/modules/storage/src/handlers/file.ts +++ b/modules/storage/src/handlers/file.ts @@ -11,7 +11,15 @@ import { status } from '@grpc/grpc-js'; import { isNil, isString } from 'lodash-es'; import { _StorageContainer, _StorageFolder, File } from '../models/index.js'; import { IStorageProvider } from '../interfaces/index.js'; -import { deepPathHandler, normalizeFolderPath } from '../utils/index.js'; +import { + _createFileUploadUrl, + _updateFile, + _updateFileUploadUrl, + deepPathHandler, + normalizeFolderPath, + storeNewFile, + validateName, +} from '../utils/index.js'; export class FileHandlers { private readonly database: DatabaseProvider; @@ -107,7 +115,7 @@ export class FileHandlers { } async createFile(call: ParsedRouterRequest): Promise { - const { name, data, container, mimeType, isPublic } = call.request.params; + const { name, alias, data, container, mimeType, isPublic } = call.request.params; await this.fileAccessCheck('create', call.request); const folder = normalizeFolderPath(call.request.params.folder); const config = ConfigController.getInstance().config; @@ -117,28 +125,17 @@ export class FileHandlers { if (folder !== '/') { await this.findOrCreateFolders(folder, usedContainer, isPublic); } - - const exists = await File.getInstance().findOne({ - name, - container: usedContainer, - folder, - }); - if (exists) { - throw new GrpcError(status.ALREADY_EXISTS, 'File already exists'); - } - if (!isString(data)) { - throw new GrpcError(status.INVALID_ARGUMENT, 'Invalid data provided'); - } - + const validatedName = await validateName(name, folder, usedContainer); try { - const file = await this.storeNewFile( + const file = await storeNewFile(this.storageProvider, { + name: validatedName, + alias, data, - usedContainer, + container: usedContainer, folder, isPublic, - name, mimeType, - ); + }); await this.fileAccessAdd(file, call.request); return file; } catch (e) { @@ -150,7 +147,7 @@ export class FileHandlers { } async createFileUploadUrl(call: ParsedRouterRequest): Promise { - const { name, container, size = 0, mimeType, isPublic } = call.request.params; + const { name, alias, container, size = 0, mimeType, isPublic } = call.request.params; await this.fileAccessCheck('create', call.request); const folder = normalizeFolderPath(call.request.params.folder); const config = ConfigController.getInstance().config; @@ -160,25 +157,17 @@ export class FileHandlers { if (folder !== '/') { await this.findOrCreateFolders(folder, usedContainer, isPublic); } - - const exists = await File.getInstance().findOne({ - name, - container: usedContainer, - folder, - }); - if (exists) { - throw new GrpcError(status.ALREADY_EXISTS, 'File already exists'); - } - + const validatedName = await validateName(name, folder, usedContainer); try { - const { file, url } = await this._createFileUploadUrl( - usedContainer, + const { file, url } = await _createFileUploadUrl(this.storageProvider, { + container: usedContainer, folder, isPublic, - name, + name: validatedName, + alias, size, mimeType, - ); + }); await this.fileAccessAdd(file, call.request); return { file, url }; } catch (e) { @@ -190,7 +179,7 @@ export class FileHandlers { } async updateFileUploadUrl(call: ParsedRouterRequest): Promise { - const { id, mimeType, size } = call.request.params; + const { id, alias, mimeType, size } = call.request.params; const found = await File.getInstance().findOne({ _id: id }); if (isNil(found)) { throw new GrpcError(status.NOT_FOUND, 'File does not exist'); @@ -201,14 +190,14 @@ export class FileHandlers { found, ); try { - return await this._updateFileUploadUrl( + return await _updateFileUploadUrl(this.storageProvider, found, { name, + alias, folder, container, - mimeType ?? found.mimeType, - found, + mimeType: mimeType ?? found.mimeType, size, - ); + }); } catch (e) { throw new GrpcError( status.INTERNAL, @@ -218,7 +207,7 @@ export class FileHandlers { } async updateFile(call: ParsedRouterRequest): Promise { - const { id, data, mimeType } = call.request.params; + const { id, alias, data, mimeType } = call.request.params; const found = await File.getInstance().findOne({ _id: id }); if (isNil(found)) { throw new GrpcError(status.NOT_FOUND, 'File does not exist'); @@ -229,14 +218,14 @@ export class FileHandlers { found, ); try { - return await this._updateFile( + return await _updateFile(this.storageProvider, found, { name, + alias, folder, container, - Buffer.from(data, 'base64'), - mimeType ?? found.mimeType, - found, - ); + data: Buffer.from(data, 'base64'), + mimeType: mimeType ?? found.mimeType, + }); } catch (e) { throw new GrpcError( status.INTERNAL, @@ -390,77 +379,6 @@ export class FileHandlers { return createdFolders; } - private async storeNewFile( - data: string, - container: string, - folder: string, - isPublic: boolean, - name: string, - mimeType: string, - ): Promise { - const buffer = Buffer.from(data, 'base64'); - const size = buffer.byteLength; - await this.storageProvider - .container(container) - .store((folder === '/' ? '' : folder) + name, buffer, isPublic); - const publicUrl = isPublic - ? await this.storageProvider - .container(container) - .getPublicUrl((folder === '/' ? '' : folder) + name) - : null; - ConduitGrpcSdk.Metrics?.increment('files_total'); - ConduitGrpcSdk.Metrics?.increment('storage_size_bytes_total', size); - return await File.getInstance().create({ - name, - mimeType, - folder: folder, - container: container, - size, - isPublic, - url: publicUrl, - }); - } - - private async _createFileUploadUrl( - container: string, - folder: string, - isPublic: boolean, - name: string, - size: number, - mimeType: string, - ): Promise<{ file: File; url: string }> { - await this.storageProvider - .container(container) - .store( - (folder === '/' ? '' : folder) + name, - Buffer.from('PENDING UPLOAD'), - isPublic, - ); - const publicUrl = isPublic - ? await this.storageProvider - .container(container) - .getPublicUrl((folder === '/' ? '' : folder) + name) - : null; - ConduitGrpcSdk.Metrics?.increment('files_total'); - ConduitGrpcSdk.Metrics?.increment('storage_size_bytes_total', size); - const file = await File.getInstance().create({ - name, - mimeType, - size, - folder: folder, - container: container, - isPublic, - url: publicUrl, - }); - const url = (await this.storageProvider - .container(container) - .getUploadUrl((folder === '/' ? '' : folder) + name)) as string; - return { - file, - url, - }; - } - private async validateFilenameAndContainer(call: ParsedRouterRequest, file: File) { const { name, folder, container } = call.request.params; const newName = name ?? file.name; @@ -490,93 +408,4 @@ export class FileHandlers { container: newContainer, }; } - - private async _updateFileUploadUrl( - name: string, - folder: string, - container: string, - mimeType: string, - file: File, - size: number | undefined | null, - ): Promise<{ file: File; url: string }> { - let updatedFile; - const onlyDataUpdate = - name === file.name && folder === file.folder && container === file.container; - if (onlyDataUpdate) { - updatedFile = await File.getInstance().findByIdAndUpdate(file._id, { - mimeType, - ...{ size: size ?? file.size }, - }); - } else { - await this.storageProvider - .container(container) - .store( - (folder === '/' ? '' : folder) + name, - Buffer.from('PENDING UPLOAD'), - file.isPublic, - ); - await this.storageProvider - .container(file.container) - .delete((file.folder === '/' ? '' : file.folder) + file.name); - const url = file.isPublic - ? await this.storageProvider - .container(container) - .getPublicUrl((folder === '/' ? '' : folder) + name) - : null; - updatedFile = await File.getInstance().findByIdAndUpdate(file._id, { - name, - folder, - container, - url, - mimeType, - ...{ size: size ?? file.size }, - }); - } - if (!isNil(size)) this.updateFileMetrics(file.size, size!); - const uploadUrl = (await this.storageProvider - .container(container) - .getUploadUrl((folder === '/' ? '' : folder) + name)) as string; - return { file: updatedFile!, url: uploadUrl }; - } - - private async _updateFile( - name: string, - folder: string, - container: string, - data: Buffer, - mimeType: string, - file: File, - ): Promise { - const onlyDataUpdate = - name === file.name && folder === file.folder && container === file.container; - await this.storageProvider - .container(container) - .store((folder === '/' ? '' : folder) + name, data, file.isPublic); - if (!onlyDataUpdate) { - await this.storageProvider - .container(file.container) - .delete((file.folder === '/' ? '' : file.folder) + file.name); - } - const url = file.isPublic - ? await this.storageProvider - .container(container) - .getPublicUrl((folder === '/' ? '' : folder) + name) - : null; - const updatedFile = (await File.getInstance().findByIdAndUpdate(file._id, { - name, - folder, - container, - url, - mimeType, - })) as File; - this.updateFileMetrics(file.size, data.byteLength); - return updatedFile; - } - - private updateFileMetrics(currentSize: number, newSize: number) { - const fileSizeDiff = Math.abs(currentSize - newSize); - fileSizeDiff < 0 - ? ConduitGrpcSdk.Metrics?.increment('storage_size_bytes_total', fileSizeDiff) - : ConduitGrpcSdk.Metrics?.decrement('storage_size_bytes_total', fileSizeDiff); - } } diff --git a/modules/storage/src/interfaces/IFileParams.ts b/modules/storage/src/interfaces/IFileParams.ts new file mode 100644 index 000000000..262e639c8 --- /dev/null +++ b/modules/storage/src/interfaces/IFileParams.ts @@ -0,0 +1,10 @@ +export interface IFileParams { + name?: string; + alias?: string; + container: string; + folder: string; + data?: string | Buffer; + isPublic?: boolean; + mimeType?: string; + size?: number; +} diff --git a/modules/storage/src/interfaces/index.ts b/modules/storage/src/interfaces/index.ts index 48d00728c..12f0b17b9 100644 --- a/modules/storage/src/interfaces/index.ts +++ b/modules/storage/src/interfaces/index.ts @@ -1,2 +1,3 @@ export * from './IStorageProvider.js'; export * from './StorageConfig.js'; +export * from './IFileParams.js'; diff --git a/modules/storage/src/models/File.schema.ts b/modules/storage/src/models/File.schema.ts index 36bf1aabf..f2f79ab89 100644 --- a/modules/storage/src/models/File.schema.ts +++ b/modules/storage/src/models/File.schema.ts @@ -5,12 +5,16 @@ const schema: ConduitModel = { _id: TYPE.ObjectId, name: { type: TYPE.String, - required: true, + required: false, }, folder: { type: TYPE.String, required: true, }, + alias: { + type: TYPE.String, + required: false, + }, container: { type: TYPE.String, required: true, @@ -46,6 +50,7 @@ export class File extends ConduitActiveSchema { _id!: string; //todo rename declare name: string; + alias: string; folder!: string; container!: string; size!: number; diff --git a/modules/storage/src/routes/index.ts b/modules/storage/src/routes/index.ts index a3dbd1a44..d48958b96 100644 --- a/modules/storage/src/routes/index.ts +++ b/modules/storage/src/routes/index.ts @@ -68,7 +68,8 @@ export class StorageRoutes { this._routingManager.route( { bodyParams: { - name: { type: TYPE.String, required: true }, + name: { type: TYPE.String, required: false }, + alias: { type: TYPE.String, required: false }, mimeType: TYPE.String, data: { type: TYPE.String, required: true }, folder: { type: TYPE.String, required: false }, @@ -89,7 +90,8 @@ export class StorageRoutes { this._routingManager.route( { bodyParams: { - name: { type: TYPE.String, required: true }, + name: { type: TYPE.String, required: false }, + alias: { type: TYPE.String, required: false }, mimeType: TYPE.String, folder: { type: TYPE.String, required: false }, size: { type: TYPE.Number, required: false }, @@ -117,6 +119,7 @@ export class StorageRoutes { }, bodyParams: { name: ConduitString.Optional, + alias: ConduitString.Optional, folder: ConduitString.Optional, container: ConduitString.Optional, mimeType: ConduitString.Optional, @@ -184,6 +187,7 @@ export class StorageRoutes { }, bodyParams: { name: ConduitString.Optional, + alias: ConduitString.Optional, folder: ConduitString.Optional, container: ConduitString.Optional, data: ConduitString.Required, diff --git a/modules/storage/src/storage.proto b/modules/storage/src/storage.proto index 6dc57138a..fc36d22a1 100644 --- a/modules/storage/src/storage.proto +++ b/modules/storage/src/storage.proto @@ -18,7 +18,7 @@ message GetFileUrlResponse { } message CreateFileRequest { - string name = 1; + optional string name = 1; string data = 2; bool isPublic = 3; optional string folder = 4; @@ -26,6 +26,7 @@ message CreateFileRequest { optional string mimeType = 6; optional string userId = 7; optional string scope = 8; + optional string alias = 9; } message UpdateFileRequest { @@ -37,6 +38,7 @@ message UpdateFileRequest { optional string mimeType = 6; optional string userId = 7; optional string scope = 8; + optional string alias = 9; } message FileResponse { @@ -53,7 +55,7 @@ message DeleteFileResponse { } message CreateFileByUrlRequest { - string name = 1; + optional string name = 1; bool isPublic = 2; optional string folder = 3; optional string container = 4; @@ -61,6 +63,7 @@ message CreateFileByUrlRequest { optional int32 size = 6; // todo: support int64 optional string userId = 7; optional string scope = 8; + optional string alias = 9; } message FileByUrlResponse { @@ -79,6 +82,7 @@ message UpdateFileByUrlRequest { optional int32 size = 6; // todo: support int64 optional string userId = 7; optional string scope = 8; + optional string alias = 9; } service Storage { diff --git a/modules/storage/src/utils/index.ts b/modules/storage/src/utils/index.ts index dbf50d71f..6b46f2ff8 100644 --- a/modules/storage/src/utils/index.ts +++ b/modules/storage/src/utils/index.ts @@ -1,7 +1,12 @@ import { GetUserCommand, IAMClient } from '@aws-sdk/client-iam'; -import { StorageConfig } from '../interfaces/index.js'; +import { IFileParams, IStorageProvider, StorageConfig } from '../interfaces/index.js'; import { isNil } from 'lodash-es'; import path from 'path'; +import { File } from '../models/index.js'; +import { ConduitGrpcSdk, GrpcError } from '@conduitplatform/grpc-sdk'; +import { randomUUID } from 'node:crypto'; +import { ConfigController } from '@conduitplatform/module-tools'; +import { status } from '@grpc/grpc-js'; export async function streamToBuffer(readableStream: any): Promise { return new Promise((resolve, reject) => { @@ -62,3 +67,184 @@ export async function deepPathHandler( await handler(paths[i], i === paths.length - 1); } } + +export async function storeNewFile( + storageProvider: IStorageProvider, + params: IFileParams, +): Promise { + const { name, alias, data, container, folder, mimeType, isPublic } = params; + const buffer = Buffer.from(data as string, 'base64'); + const size = buffer.byteLength; + const fileName = (folder === '/' ? '' : folder) + name; + await storageProvider.container(container).store(fileName, buffer, isPublic); + const publicUrl = isPublic + ? await storageProvider.container(container).getPublicUrl(fileName) + : null; + ConduitGrpcSdk.Metrics?.increment('files_total'); + ConduitGrpcSdk.Metrics?.increment('storage_size_bytes_total', size); + return await File.getInstance().create({ + name, + alias, + mimeType, + folder: folder, + container: container, + size, + isPublic, + url: publicUrl, + }); +} + +export async function _createFileUploadUrl( + storageProvider: IStorageProvider, + params: IFileParams, +): Promise<{ file: File; url: string }> { + const { name, alias, container, folder, mimeType, isPublic, size } = params; + const fileName = (folder === '/' ? '' : folder) + name; + await storageProvider + .container(container) + .store(fileName, Buffer.from('PENDING UPLOAD'), isPublic); + const publicUrl = isPublic + ? await storageProvider.container(container).getPublicUrl(fileName) + : null; + ConduitGrpcSdk.Metrics?.increment('files_total'); + ConduitGrpcSdk.Metrics?.increment('storage_size_bytes_total', size); + const file = await File.getInstance().create({ + name, + alias, + mimeType, + size, + folder: folder, + container: container, + isPublic, + url: publicUrl, + }); + const url = (await storageProvider + .container(container) + .getUploadUrl(fileName)) as string; + return { + file, + url, + }; +} + +export async function _updateFile( + storageProvider: IStorageProvider, + file: File, + params: IFileParams, +): Promise { + const { name, alias, data, folder, container, mimeType } = params; + const onlyDataUpdate = + name === file.name && folder === file.folder && container === file.container; + await storageProvider + .container(container) + .store((folder === '/' ? '' : folder) + name, data, file.isPublic); + if (!onlyDataUpdate) { + await storageProvider + .container(file.container) + .delete((file.folder === '/' ? '' : file.folder) + file.name); + } + const url = file.isPublic + ? await storageProvider + .container(container) + .getPublicUrl((folder === '/' ? '' : folder) + name) + : null; + const updatedFile = (await File.getInstance().findByIdAndUpdate(file._id, { + name, + alias, + folder, + container, + url, + mimeType, + })) as File; + updateFileMetrics(file.size, (data as Buffer).byteLength); + return updatedFile; +} + +export async function _updateFileUploadUrl( + storageProvider: IStorageProvider, + file: File, + params: IFileParams, +): Promise<{ file: File; url: string }> { + const { name, alias, folder, container, mimeType, size } = params; + let updatedFile; + const onlyDataUpdate = + name === file.name && folder === file.folder && container === file.container; + if (onlyDataUpdate) { + updatedFile = await File.getInstance().findByIdAndUpdate(file._id, { + mimeType, + alias, + ...{ size: size ?? file.size }, + }); + } else { + await storageProvider + .container(container) + .store( + (folder === '/' ? '' : folder) + name, + Buffer.from('PENDING UPLOAD'), + file.isPublic, + ); + await storageProvider + .container(file.container) + .delete((file.folder === '/' ? '' : file.folder) + file.name); + const url = file.isPublic + ? await storageProvider + .container(container) + .getPublicUrl((folder === '/' ? '' : folder) + name) + : null; + updatedFile = await File.getInstance().findByIdAndUpdate(file._id, { + name, + alias, + folder, + container, + url, + mimeType, + ...{ size: size ?? file.size }, + }); + } + if (!isNil(size)) updateFileMetrics(file.size, size!); + const uploadUrl = (await storageProvider + .container(container) + .getUploadUrl((folder === '/' ? '' : folder) + name)) as string; + return { file: updatedFile!, url: uploadUrl }; +} + +export function updateFileMetrics(currentSize: number, newSize: number) { + const fileSizeDiff = Math.abs(currentSize - newSize); + fileSizeDiff < 0 + ? ConduitGrpcSdk.Metrics?.increment('storage_size_bytes_total', fileSizeDiff) + : ConduitGrpcSdk.Metrics?.decrement('storage_size_bytes_total', fileSizeDiff); +} + +export async function validateName( + name: string | undefined, + folder: string, + container: string, +) { + if (!name) { + return randomUUID(); + } + const config = ConfigController.getInstance().config; + const extension = path.extname(name); + const escapedExtension = extension.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&'); + const baseName = path.basename(name, extension); + const escapedBaseName = baseName.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&'); + const regexPattern = `^${escapedBaseName} \\(\\d+\\)${escapedExtension}$`; + + const count = await File.getInstance().countDocuments({ + $and: [ + { $or: [{ name }, { name: { $regex: regexPattern } }] }, + { folder: folder }, + { container: container }, + ], + }); + if (count === 0) { + return name; + } else if (!config.suffixOnNameConflict) { + throw new GrpcError(status.ALREADY_EXISTS, 'File already exists'); + } else { + if (extension !== '') { + return `${baseName} (${count})${extension}`; + } + return `${name} (${count})`; + } +}