diff --git a/.github/workflows/prune-images.yml b/.github/workflows/prune-images.yml new file mode 100644 index 000000000..af914dc9d --- /dev/null +++ b/.github/workflows/prune-images.yml @@ -0,0 +1,19 @@ +name: prune-images + +on: + schedule: + - cron: "0 0 1 * *" # Runs at 00:00, on day 1 of the month + workflow_dispatch: # Can also be run manually from the github UI + +jobs: + request_pruning: + name: Request images pruning + runs-on: ubuntu-latest + + steps: + - env: + SUPER_SECRET: ${{ secrets.PRUNING_ACCESS_TOKEN }} + run: | + curl 'https://ara.numerique.gouv.fr/api/system/prune-uploads' + -H 'X-PruneToken: Bearer "$PRUNING_ACCESS_TOKEN"' + -d '' diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..e61bd3c26 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,40 @@ +{ + "version": "0.2.0", + "configurations": [ + // Frontend (Chrome) + { + "type": "chrome", + "request": "launch", + "name": "FRONT (Vue.js) – Chrome", + "url": "http://localhost:3000", + "webRoot": "${workspaceFolder}/confiture-web-app/src", + "sourceMapPathOverrides": { + "webpack:///src/*": "${webRoot}/*" + } + }, + // Frontend (Chrome) + { + "type": "firefox", + "request": "launch", + "name": "FRONT (Vue.js) – Firefox", + "url": "http://localhost:3000", + "webRoot": "${workspaceFolder}/confiture-web-app/src", + "pathMappings": [ + { "url": "webpack:///confiture-web-app/src/", "path": "${webRoot}/" } + ] + }, + // Backend (REST API) + { + "type": "node", + "request": "launch", + "name": "BACKEND (nest)", + "runtimeExecutable": "npm", + "runtimeArgs": ["run", "start:debug", "--", "--inspect-brk"], + "autoAttachChildProcesses": true, + "restart": true, + "sourceMaps": true, + "stopOnEntry": false, + "console": "integratedTerminal" + } + ] +} diff --git a/confiture-rest-api/.env.example b/confiture-rest-api/.env.example index 83803b2e2..54fbfea07 100644 --- a/confiture-rest-api/.env.example +++ b/confiture-rest-api/.env.example @@ -19,4 +19,8 @@ S3_VIRTUAL_HOST="xxx" AWS_ACCESS_KEY_ID="xxx" AWS_SECRET_ACCESS_KEY="xxx" -JWT_SECRET="xxx" \ No newline at end of file +JWT_SECRET="xxx" + +# Debug Prisma queries +# More info: [Logging | Prisma Documentation](https://www.prisma.io/docs/orm/prisma-client/observability-and-logging/logging) +DEBUG="prisma:query" diff --git a/confiture-rest-api/prisma/migrations/20240906161141_add_display_field_on_files_attachment_or_editor/migration.sql b/confiture-rest-api/prisma/migrations/20240906161141_add_display_field_on_files_attachment_or_editor/migration.sql new file mode 100644 index 000000000..503297a73 --- /dev/null +++ b/confiture-rest-api/prisma/migrations/20240906161141_add_display_field_on_files_attachment_or_editor/migration.sql @@ -0,0 +1,8 @@ +-- CreateEnum +CREATE TYPE "FileDisplay" AS ENUM ('EDITOR', 'ATTACHMENT'); + +-- AlterTable +ALTER TABLE "AuditFile" ADD COLUMN "display" "FileDisplay" NOT NULL DEFAULT 'ATTACHMENT'; + +-- AlterTable +ALTER TABLE "StoredFile" ADD COLUMN "display" "FileDisplay" NOT NULL DEFAULT 'ATTACHMENT'; diff --git a/confiture-rest-api/prisma/migrations/20250121151450_add_audit_file_creation_date/migration.sql b/confiture-rest-api/prisma/migrations/20250121151450_add_audit_file_creation_date/migration.sql new file mode 100644 index 000000000..7a3efe9f8 --- /dev/null +++ b/confiture-rest-api/prisma/migrations/20250121151450_add_audit_file_creation_date/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "AuditFile" ADD COLUMN "creationDate" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP; diff --git a/confiture-rest-api/prisma/migrations/20250124221428_add_stored_file_creation_date/migration.sql b/confiture-rest-api/prisma/migrations/20250124221428_add_stored_file_creation_date/migration.sql new file mode 100644 index 000000000..a26845837 --- /dev/null +++ b/confiture-rest-api/prisma/migrations/20250124221428_add_stored_file_creation_date/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "StoredFile" ADD COLUMN "creationDate" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP; diff --git a/confiture-rest-api/prisma/migrations/20250126234052_thumbnail_optional_for_stored_file/migration.sql b/confiture-rest-api/prisma/migrations/20250126234052_thumbnail_optional_for_stored_file/migration.sql new file mode 100644 index 000000000..9a47236da --- /dev/null +++ b/confiture-rest-api/prisma/migrations/20250126234052_thumbnail_optional_for_stored_file/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "StoredFile" ALTER COLUMN "thumbnailKey" DROP NOT NULL; diff --git a/confiture-rest-api/prisma/schema.prisma b/confiture-rest-api/prisma/schema.prisma index 3c1a51b21..35b965e5e 100644 --- a/confiture-rest-api/prisma/schema.prisma +++ b/confiture-rest-api/prisma/schema.prisma @@ -169,6 +169,11 @@ model AuditTrace { Audit Audit? } +enum FileDisplay { + EDITOR + ATTACHMENT +} + model StoredFile { id Int @id @default(autoincrement()) originalFilename String @@ -182,7 +187,12 @@ model StoredFile { // S3 storage keys key String - thumbnailKey String + thumbnailKey String? + + // Inside TipTap editor (EDITOR) or added as an attachment (ATTACHMENT)? + display FileDisplay @default(ATTACHMENT) + + creationDate DateTime @default(now()) criterionResult CriterionResult? @relation(fields: [criterionResultId], references: [id], onDelete: Cascade, onUpdate: Cascade) criterionResultId Int? @@ -203,6 +213,11 @@ model AuditFile { key String thumbnailKey String? + creationDate DateTime @default(now()) + + // Inside TipTap editor (EDITOR) or added as an attachment (ATTACHMENT)? + display FileDisplay @default(ATTACHMENT) + audit Audit? @relation(fields: [auditUniqueId], references: [editUniqueId], onDelete: Cascade) auditUniqueId String? } diff --git a/confiture-rest-api/src/app.module.ts b/confiture-rest-api/src/app.module.ts index 149f65d6a..8553a91e3 100644 --- a/confiture-rest-api/src/app.module.ts +++ b/confiture-rest-api/src/app.module.ts @@ -7,6 +7,7 @@ import { configValidationSchema } from "./config-validation-schema"; import { MailModule } from "./mail/mail.module"; import { AuthModule } from "./auth/auth.module"; import { ProfileModule } from "./profile/profile.module"; +import { SystemModule } from "./system/system.module"; import { UserMiddleware } from "./auth/user.middleware"; @Module({ @@ -19,7 +20,8 @@ import { UserMiddleware } from "./auth/user.middleware"; AuditsModule, MailModule, AuthModule, - ProfileModule + ProfileModule, + SystemModule ], controllers: [HealthCheckController] }) diff --git a/confiture-rest-api/src/audits/audit.service.ts b/confiture-rest-api/src/audits/audit.service.ts index f613c9e82..198ad3867 100644 --- a/confiture-rest-api/src/audits/audit.service.ts +++ b/confiture-rest-api/src/audits/audit.service.ts @@ -4,6 +4,7 @@ import { CriterionResult, CriterionResultStatus, CriterionResultUserImpact, + FileDisplay, Prisma, StoredFile } from "@prisma/client"; @@ -418,30 +419,14 @@ export class AuditService { pageId: number, topic: number, criterium: number, - file: Express.Multer.File + file: Express.Multer.File, + display: FileDisplay = FileDisplay.ATTACHMENT ) { - const randomPrefix = nanoid(); - - const key = `audits/${editUniqueId}/${randomPrefix}/${file.originalname}`; - - const thumbnailKey = `audits/${editUniqueId}/${randomPrefix}/thumbnail_${file.originalname}`; - - const thumbnailBuffer = await sharp(file.buffer) - .jpeg({ - mozjpeg: true - }) - .flatten({ background: { r: 255, g: 255, b: 255, alpha: 0 } }) - .resize(200, 200, { fit: "inside" }) - .toBuffer(); - - await Promise.all([ - this.fileStorageService.uploadFile(file.buffer, file.mimetype, key), - this.fileStorageService.uploadFile( - thumbnailBuffer, - "image/jpeg", - thumbnailKey - ) - ]); + const { key, thumbnailKey } = await this.uploadFileToStorage( + editUniqueId, + file, + { createThumbnail: display === FileDisplay.ATTACHMENT } + ); const storedFile = await this.prisma.storedFile.create({ data: { @@ -459,7 +444,8 @@ export class AuditService { originalFilename: file.originalname, mimetype: file.mimetype, size: file.size, - thumbnailKey + thumbnailKey, + display } }); @@ -500,22 +486,59 @@ export class AuditService { return true; } - async saveNotesFile(editUniqueId: string, file: Express.Multer.File) { + async saveNotesFile( + editUniqueId: string, + file: Express.Multer.File, + display: FileDisplay = FileDisplay.ATTACHMENT + ) { + const { key, thumbnailKey } = await this.uploadFileToStorage( + editUniqueId, + file, + { createThumbnail: display === FileDisplay.ATTACHMENT } + ); + + const storedFile = await this.prisma.auditFile.create({ + data: { + audit: { + connect: { + editUniqueId + } + }, + + key, + originalFilename: file.originalname, + mimetype: file.mimetype, + size: file.size, + + thumbnailKey, + display + } + }); + + return storedFile; + } + + async uploadFileToStorage( + uniqueId: string, + file: Express.Multer.File, + options?: { createThumbnail: boolean } + ): Promise<{ key: string; thumbnailKey?: string }> { const randomPrefix = nanoid(); - const key = `audits/${editUniqueId}/${randomPrefix}/${file.originalname}`; + const key: string = `audits/${uniqueId}/${randomPrefix}/${file.originalname}`; - let thumbnailKey; + let thumbnailKey: string; - if (file.mimetype.startsWith("image")) { + if (file.mimetype.startsWith("image") && options.createThumbnail) { // If it's an image, create a thumbnail and upload it - thumbnailKey = `audits/${editUniqueId}/${randomPrefix}/thumbnail_${file.originalname}`; + thumbnailKey = `audits/${uniqueId}/${randomPrefix}/thumbnail_${file.originalname}`; const thumbnailBuffer = await sharp(file.buffer) - .resize(200, 200, { fit: "inside" }) .jpeg({ mozjpeg: true }) + .flatten({ background: { r: 255, g: 255, b: 255, alpha: 0 } }) + .resize(200, 200, { fit: "inside" }) .toBuffer(); await Promise.all([ @@ -529,29 +552,11 @@ export class AuditService { } else { await this.fileStorageService.uploadFile(file.buffer, file.mimetype, key); } - - const storedFile = await this.prisma.auditFile.create({ - data: { - audit: { - connect: { - editUniqueId - } - }, - - key, - originalFilename: file.originalname, - mimetype: file.mimetype, - size: file.size, - - thumbnailKey - } - }); - - return storedFile; + return { key, thumbnailKey }; } /** - * Returns true if stored filed was found and deleted. False if not found. + * Returns true if stored file has been found and deleted. False if not found. */ async deleteAuditFile( editUniqueId: string, @@ -585,6 +590,42 @@ export class AuditService { return true; } + /** + * Returns true if all stored file have been found and deleted. False otherwise. + */ + async deleteAuditFiles(fileIds: number[]): Promise { + const storedFiles = await this.prisma.auditFile.findMany({ + select: { + id: true, + key: true, + thumbnailKey: true + }, + where: { + id: { + in: fileIds + } + } + }); + + const filesToDelete = storedFiles.map((e) => e.key); + const thumbnailsToDelete = storedFiles + .map((e) => e.thumbnailKey) + .filter((e) => e != null); + await this.fileStorageService.deleteMultipleFiles( + ...filesToDelete.concat(thumbnailsToDelete) + ); + + await this.prisma.auditFile.deleteMany({ + where: { + id: { + in: fileIds + } + } + }); + + return true; + } + /** * Completely delete an audit and all the data associated with it. * @returns True if an audit was deleted, false otherwise. @@ -832,7 +873,8 @@ export class AuditService { key: file.key, thumbnailKey: file.thumbnailKey, size: file.size, - mimetype: file.mimetype + mimetype: file.mimetype, + display: file.display })), criteriaCount: { @@ -999,7 +1041,8 @@ export class AuditService { exampleImages: r.exampleImages.map((img) => ({ filename: img.originalFilename, key: img.key, - thumbnailKey: img.thumbnailKey + thumbnailKey: img.thumbnailKey, + display: img.display })) })) }; diff --git a/confiture-rest-api/src/audits/audits.controller.ts b/confiture-rest-api/src/audits/audits.controller.ts index d2a7c92af..391737ae4 100644 --- a/confiture-rest-api/src/audits/audits.controller.ts +++ b/confiture-rest-api/src/audits/audits.controller.ts @@ -28,6 +28,7 @@ import { import { Audit } from "src/generated/nestjs-dto/audit.entity"; import { CriterionResult } from "src/generated/nestjs-dto/criterionResult.entity"; import { MailService } from "../mail/mail.service"; +import { NotesFileDto } from "./dto/notes-file.dto"; import { AuditExportService } from "./audit-export.service"; import { AuditService } from "./audit.service"; import { CreateAuditDto } from "./dto/create-audit.dto"; @@ -174,7 +175,8 @@ export class AuditsController { body.pageId, body.topic, body.criterium, - file + file, + body.display ); } @@ -191,7 +193,8 @@ export class AuditsController { errorHttpStatusCode: HttpStatus.UNPROCESSABLE_ENTITY }) ) - file: Express.Multer.File + file: Express.Multer.File, + @Body() body: NotesFileDto ) { const audit = await this.auditService.getAuditWithEditUniqueId(uniqueId); @@ -199,7 +202,7 @@ export class AuditsController { return this.sendAuditNotFoundStatus(uniqueId); } - return await this.auditService.saveNotesFile(uniqueId, file); + return await this.auditService.saveNotesFile(uniqueId, file, body.display); } @Delete("/:uniqueId/results/examples/:exampleId") diff --git a/confiture-rest-api/src/audits/dto/audit-report.dto.ts b/confiture-rest-api/src/audits/dto/audit-report.dto.ts index 0ac3e5be5..ecc7a5d4f 100644 --- a/confiture-rest-api/src/audits/dto/audit-report.dto.ts +++ b/confiture-rest-api/src/audits/dto/audit-report.dto.ts @@ -2,7 +2,8 @@ import { ApiProperty } from "@nestjs/swagger"; import { AuditType, CriterionResultStatus, - CriterionResultUserImpact + CriterionResultUserImpact, + FileDisplay } from "@prisma/client"; export class AuditReportDto { @@ -190,18 +191,35 @@ class ReportCriterionResult { } class ExampleImage { - /** @example "mon-image.jpg" */ - filename: string; - /** @example "audit/xxxx/my-image.jpg" */ + /** @example "screenshot_001.png" */ + originalFilename?: string; + /** @example "audits/EWIsM6sYI2cC0lI7Ok2PE/uqoOes4QqhFyKV8v0s2AQ/screenshot_001.png" */ key: string; - /** @example "audit/xxxx/my-image_thumbnail.jpg" */ + /** @example "audits/EWIsM6sYI2cC0lI7Ok2PE/uqoOes4QqhFyKV8v0s2AQ/thumbnail_screenshot_001.png" */ thumbnailKey: string; + /** @example 4631 */ + size?: number; + /** @example "image/png" */ + mimetype?: string; + /** @example "ATTACHMENT" */ + display: FileDisplay; + /** @example 2025-01-24T16:39:12.811Z */ + creationDate?: Date; } class NotesFile { - originalFilename: string; + /** @example "screenshot_001.png" */ + originalFilename?: string; + /** @example "audits/EWIsM6sYI2cC0lI7Ok2PE/uqoOes4QqhFyKV8v0s2AQ/screenshot_001.png" */ key: string; + /** @example "audits/EWIsM6sYI2cC0lI7Ok2PE/uqoOes4QqhFyKV8v0s2AQ/thumbnail_screenshot_001.png" */ thumbnailKey: string; - size: number; - mimetype: string; + /** @example 4631 */ + size?: number; + /** @example "image/png" */ + mimetype?: string; + /** @example "ATTACHMENT" */ + display: FileDisplay; + /** @example 2025-01-24T16:39:12.811Z */ + creationDate?: Date; } diff --git a/confiture-rest-api/src/audits/dto/notes-file.dto.ts b/confiture-rest-api/src/audits/dto/notes-file.dto.ts new file mode 100644 index 000000000..8e3bc9cbf --- /dev/null +++ b/confiture-rest-api/src/audits/dto/notes-file.dto.ts @@ -0,0 +1,9 @@ +import { FileDisplay } from "@prisma/client"; +import { IsIn, IsOptional, IsString } from "class-validator"; + +export class NotesFileDto { + @IsOptional() + @IsString() + @IsIn(Object.values(FileDisplay)) + display?: FileDisplay; +} diff --git a/confiture-rest-api/src/audits/dto/upload-image.dto.ts b/confiture-rest-api/src/audits/dto/upload-image.dto.ts index 810f58478..9c48b352a 100644 --- a/confiture-rest-api/src/audits/dto/upload-image.dto.ts +++ b/confiture-rest-api/src/audits/dto/upload-image.dto.ts @@ -1,6 +1,16 @@ import { Type } from "class-transformer"; -import { IsInt, IsNumber, IsPositive, Max, Min } from "class-validator"; +import { + IsIn, + IsInt, + IsNumber, + IsOptional, + IsPositive, + IsString, + Max, + Min +} from "class-validator"; import { IsRgaaCriterium } from "./update-results.dto"; +import { FileDisplay } from "@prisma/client"; /* The `@Type(() => Number)` decorator is required to correctly parse strings into numbers @@ -34,4 +44,9 @@ export class UploadImageDto { "topic and criterium numbers must be a valid RGAA criterium combination" }) criterium: number; + + @IsOptional() + @IsString() + @IsIn(Object.values(FileDisplay)) + display: FileDisplay; } diff --git a/confiture-rest-api/src/audits/file-storage.service.ts b/confiture-rest-api/src/audits/file-storage.service.ts index b5b44725a..f0b50fde1 100644 --- a/confiture-rest-api/src/audits/file-storage.service.ts +++ b/confiture-rest-api/src/audits/file-storage.service.ts @@ -3,7 +3,9 @@ import { PutObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, - CopyObjectCommand + CopyObjectCommand, + ListObjectsV2Command, + ListObjectsV2Output } from "@aws-sdk/client-s3"; import { Injectable } from "@nestjs/common"; import { ConfigService } from "@nestjs/config"; @@ -34,6 +36,40 @@ export class FileStorageService { await this.s3Client.send(command); } + /** + * Retrieves all the keys of the files stored in the S3 bucket. + * Note: it’s retrieved by chunks of 1000 files at a time. + * + * @returns {Promise} An array of strings representing the keys of the files stored in the S3 bucket. + */ + async getAllFileKeys() { + let allFiles = []; + let shouldContinue = true; + let nextContinuationToken = null; + let command = null; + while (shouldContinue) { + command = new ListObjectsV2Command({ + Bucket: this.config.get("S3_BUCKET"), + ContinuationToken: nextContinuationToken || undefined + }); + + const res: ListObjectsV2Output = await (( + this.s3Client.send(command) + )); + if (!res.Contents?.length) { + break; + } + allFiles = [...allFiles, ...res.Contents]; + if (res.IsTruncated) { + nextContinuationToken = res.NextContinuationToken; + } else { + shouldContinue = false; + nextContinuationToken = null; + } + } + return allFiles.map((e) => e.Key); + } + getPublicUrl(key: string): string { return `${this.config.get("FRONT_BASE_URL")}/${key}}`; } diff --git a/confiture-rest-api/src/config-validation-schema.ts b/confiture-rest-api/src/config-validation-schema.ts index e12720cde..1dc670769 100644 --- a/confiture-rest-api/src/config-validation-schema.ts +++ b/confiture-rest-api/src/config-validation-schema.ts @@ -34,5 +34,6 @@ export const configValidationSchema = Joi.object({ .required(), AWS_ACCESS_KEY_ID: Joi.string().required(), AWS_SECRET_ACCESS_KEY: Joi.string().required(), - JWT_SECRET: Joi.string().required() + JWT_SECRET: Joi.string().required(), + PRUNING_ACCESS_TOKEN: Joi.string().min(32).required() }); diff --git a/confiture-rest-api/src/prisma.service.ts b/confiture-rest-api/src/prisma.service.ts index aaa764a18..473e6ef67 100644 --- a/confiture-rest-api/src/prisma.service.ts +++ b/confiture-rest-api/src/prisma.service.ts @@ -1,10 +1,47 @@ import { INestApplication, Injectable, OnModuleInit } from "@nestjs/common"; -import { PrismaClient } from "@prisma/client"; +import { Prisma, PrismaClient } from "@prisma/client"; @Injectable() -export class PrismaService extends PrismaClient implements OnModuleInit { +export class PrismaService + extends PrismaClient< + Prisma.PrismaClientOptions, + "query" | "info" | "warn" | "error" + > + implements OnModuleInit +{ + constructor() { + const debugQuery = process.env.DEBUG === "prisma:query"; + let logObject = {}; + if (debugQuery) { + // Note: ideally we would use `emit: "event"` instead of `emit: "stdout"` + // to avoid double logging, but Prisma's query logging is connection-scoped + // rather than query-scoped, so queries using an existing connection + // would not appear in logs + // TODO: improve when upgrading Prisma + logObject = { + log: [ + { + emit: "stdout", + level: "query" + } + ] + }; + } + super(logObject); + } + async onModuleInit() { await this.$connect(); + this.$on("query", (query: Prisma.QueryEvent) => { + let q = query.query; + JSON.parse(query.params).forEach((e, i) => { + q = q.replace(`$${i + 1}`, `'${e}'`); + }); + console.log("======================================="); + console.log("--- Prisma Query (with $n replaced) ---"); + console.log(q); + console.log("Duration: " + query.duration + "ms\n"); + }); } async enableShutdownHooks(app: INestApplication) { diff --git a/confiture-rest-api/src/system/system.controller.ts b/confiture-rest-api/src/system/system.controller.ts new file mode 100644 index 000000000..4cb3440fa --- /dev/null +++ b/confiture-rest-api/src/system/system.controller.ts @@ -0,0 +1,55 @@ +import { + Controller, + HttpCode, + Post, + Req, + UnauthorizedException +} from "@nestjs/common"; +import { + ApiHeader, + ApiOkResponse, + ApiUnauthorizedResponse +} from "@nestjs/swagger"; +import { SystemService } from "./system.service"; +import { ConfigService } from "@nestjs/config"; +import { Request } from "express"; + +@Controller("system") +export class SystemController { + constructor( + private readonly systemService: SystemService, + private readonly config: ConfigService + ) {} + + /** + * Prune "expired" uploads + * ("expired" = removed from rich text editors + old enough to avoid undo/redo issues) + * + * Removes: + * - StoredFile (criteria) and AuditFile (notes) entries + * - corresponfing files from the S3 bucket + */ + @Post("prune-uploads") + @HttpCode(200) + @ApiHeader({ + name: "X-PruneToken", + example: "Bearer abc123" + }) + @ApiOkResponse({ description: "Expired uploads pruned successfully" }) + @ApiUnauthorizedResponse({ description: "Invalid access token." }) + async pruneUploads(@Req() req: Request) { + // Check access token and return 401 in case of mismatch + { + const expectedToken = this.config.get("PRUNING_ACCESS_TOKEN"); + const requestToken = /^Bearer (.+)$/ + .exec(req.headers["x-prunetoken"] as string | undefined) + ?.at(1); + + if (expectedToken !== requestToken) { + throw new UnauthorizedException(); + } + } + + await this.systemService.pruneUploads(); + } +} diff --git a/confiture-rest-api/src/system/system.module.ts b/confiture-rest-api/src/system/system.module.ts new file mode 100644 index 000000000..33659af02 --- /dev/null +++ b/confiture-rest-api/src/system/system.module.ts @@ -0,0 +1,13 @@ +import { Module } from "@nestjs/common"; +import { SystemService } from "./system.service"; +import { SystemController } from "./system.controller"; +import { PrismaService } from "src/prisma.service"; +import { FileStorageService } from "src/audits/file-storage.service"; + +@Module({ + // FIXME: put PrismaService into a global module so the service is not instanciated multiple times + providers: [SystemService, PrismaService, FileStorageService], + controllers: [SystemController], + exports: [SystemService] +}) +export class SystemModule {} diff --git a/confiture-rest-api/src/system/system.service.ts b/confiture-rest-api/src/system/system.service.ts new file mode 100644 index 000000000..6695c9dbc --- /dev/null +++ b/confiture-rest-api/src/system/system.service.ts @@ -0,0 +1,222 @@ +import { Injectable } from "@nestjs/common"; +import { Prisma } from "@prisma/client"; +import { PrismaService } from "src/prisma.service"; +import { FileStorageService } from "../audits/file-storage.service"; + +const FileType = { + NOTES: "NOTES", + CRITERIA: "CRITERIA" +}; + +type FileType = (typeof FileType)[keyof typeof FileType]; + +type PrunableFile = { + fileType: FileType; + id: number; + key: string; + thumbnailKey: string; +}; + +@Injectable() +export class SystemService { + constructor( + private readonly prisma: PrismaService, + private readonly fileStorageService: FileStorageService + ) {} + + /** + * Deletes expired AuditFile and StoredFile entries from the database + the associated files from the S3 bucket. + * For each Audit, checks if the file URLs are still present in: + * - notes (AuditFile entries) + * - all criterionResult comments (StoredFile entries) + * If the URL is found, the AuditFile (or StoredFile) is considered expired only if it has been created more than 1 month ago. + * In that case, the entry is deleted from the database and the associated files (1 or 2 if a thumbnail exists) are deleted from the S3 bucket. + * + * Also, checks for obsolete files on the S3 bucket that are not associated anymore to any entry in the database, + * and deletes them if needed (checks all notes and all criteria in the database). + * + * Good to know: A key looks like "audits/lOuFFlopCxZ_mLKzAqpzu/BN2Jhq-iOiTIG96-f3lQU/image.png" + * It contains the audit id, a specific key for the file and the file name + * (or "external" if the image comes from an external URL – i.e. dragged and dropped + * from another website). + */ + async pruneUploads() { + console.info(`1. Check for expired AuditFile/StoredFile entries on Ara DB`); + const query = Prisma.sql` + SELECT ${FileType.NOTES} as "fileType", "AuditFile"."id", "AuditFile"."key", "AuditFile"."thumbnailKey" FROM "AuditFile" + JOIN "Audit" ON "AuditFile"."auditUniqueId" = "Audit"."editUniqueId" + WHERE ( + "AuditFile"."display" = 'EDITOR' + AND + "Audit"."notes" !~ ('"/uploads/' || "AuditFile"."key" || '"') + AND + "AuditFile"."creationDate" < now() - interval '1 month' + ) + UNION + SELECT ${FileType.CRITERIA} as "fileType", "StoredFile"."id", "StoredFile"."key", "StoredFile"."thumbnailKey" FROM "StoredFile" + JOIN "CriterionResult" ON "StoredFile"."criterionResultId" = "CriterionResult"."id" + WHERE ( + "StoredFile"."display" = 'EDITOR' + AND + CONCAT_WS('', + "CriterionResult"."compliantComment", + "CriterionResult"."errorDescription", + "CriterionResult"."notApplicableComment") !~ ('"/uploads/' || "StoredFile"."key" || '"') + AND + "StoredFile"."creationDate" < now() - interval '1 month' + )`; + const prunableUploads: PrunableFile[] = await this.prisma.$queryRaw(query); + + if (prunableUploads.length > 0) { + const entries = prunableUploads.length > 1 ? "entries" : "entry"; + console.info(` 🗑 ${prunableUploads.length} expired ${entries} found!`); + console.info(` → ${prunableUploads.map((e) => e.id).join(", ")}`); + + const oldImgs = prunableUploads.map((e) => e.key); + const oldThmbs = + prunableUploads.map((e) => e.thumbnailKey).filter((e) => e) || []; + await this.fileStorageService.deleteMultipleFiles( + ...oldImgs.concat(oldThmbs) + ); + const fS = oldImgs.length > 1 ? "s" : ""; + const fIcon = oldImgs.length > 0 ? "✅" : "🙅"; + const tS = oldThmbs.length > 1 ? "s" : ""; + const tIcon = oldThmbs.length > 0 ? "✅" : "🙅"; + console.info(` a) S3 Bucket:`); + console.info(` ${fIcon} ${oldImgs.length} file${fS} deleted`); + if (oldImgs.length > 0) { + console.info(` → key${fS}: ${oldImgs.join(", ")}`); + } + console.info(` ${tIcon} ${oldThmbs.length} thumbnail${tS} deleted`); + if (oldThmbs.length > 0) { + console.info(` → key${tS}: ${oldThmbs.join(", ")}`); + } + + const prunableAuditFileIds = prunableUploads + .filter((e) => e.fileType === FileType.NOTES) + .map((e) => e.id); + await this.prisma.auditFile.deleteMany({ + where: { + id: { + in: prunableAuditFileIds + } + } + }); + const prunableStoredFileIds = prunableUploads + .filter((e) => e.fileType === FileType.CRITERIA) + .map((e) => e.id); + await this.prisma.storedFile.deleteMany({ + where: { + id: { + in: prunableStoredFileIds + } + } + }); + console.info(` b) Ara DB:`); + const aIcon = prunableAuditFileIds.length > 0 ? "✅" : "🙅"; + const sIcon = prunableStoredFileIds.length > 0 ? "✅" : "🙅"; + console.info( + ` ${aIcon} ${prunableAuditFileIds.length} expired AuditFile ${entries} deleted` + ); + if (prunableAuditFileIds.length > 0) { + console.info(` → ${prunableAuditFileIds.join(", ")}`); + } + console.info( + ` ${sIcon} ${prunableStoredFileIds.length} expired StoredFile ${entries} deleted` + ); + if (prunableStoredFileIds.length > 0) { + console.info(` → ${prunableStoredFileIds.join(", ")}`); + } + } else { + console.info(` 🙅 No expired entry found.`); + } + + const keyOnS3 = await this.fileStorageService.getAllFileKeys(); + if (keyOnS3) { + const s = keyOnS3.length > 1 ? "s" : ""; + console.info(`2. Check for obsolete images on S3 bucket`); + console.info(` → total = ${keyOnS3.length} file${s}:`); + console.info(`${keyOnS3.join("\n")}`); + const obsoleteKeys: string[] = []; + const basePathLength = "audits/".length; + let auditUniqueId = null; + let res = null; + let url = null; + let query = null; + for (const key of keyOnS3) { + // Extract audit id from key + auditUniqueId = key.substring( + basePathLength, + key.indexOf("/", basePathLength) + ); + url = `"/uploads/${key}"`; + + // Query to check if the key is still present in notes + // or if the AuditFile/StoredFile is too recent (less than 1 month) + // That last condition is usefull for undo/redos. + query = Prisma.sql` + SELECT 1 FROM "AuditFile" + JOIN "Audit" ON "AuditFile"."auditUniqueId" = "Audit"."editUniqueId" + WHERE ( ( + "Audit"."editUniqueId" = ${auditUniqueId} + AND + "Audit"."notes" ~ ${url} + ) + OR ( + ("AuditFile"."key" = ${key} OR "AuditFile"."thumbnailKey" = ${key}) + AND + "AuditFile"."creationDate" >= now() - interval '1 month' + ) + ) + UNION + SELECT 1 FROM "StoredFile" + JOIN "CriterionResult" ON "StoredFile"."criterionResultId" = "CriterionResult"."id" + JOIN "AuditedPage" ON "CriterionResult"."pageId" = "AuditedPage"."id" + JOIN "Audit" ON ( + "AuditedPage"."auditUniqueId" = "Audit"."editUniqueId" + OR + "AuditedPage"."id" = "Audit"."transverseElementsPageId" + ) + WHERE ( ( + "Audit"."editUniqueId" = ${auditUniqueId} + AND + CONCAT_WS('', "CriterionResult"."compliantComment", "CriterionResult"."errorDescription", "CriterionResult"."notApplicableComment") ~ ${url} + ) + OR + ( + "StoredFile"."key" = ${key} OR "StoredFile"."thumbnailKey" = ${key} + AND + "StoredFile"."creationDate" >= now() - interval '1 month' + ) )`; + + res = await this.prisma.$queryRaw(query); + if (res.length === 0) { + console.warn( + ` Key "${key}" not found for audit "${auditUniqueId}" → mark it as obsolete"` + ); + obsoleteKeys.push(key); + } + } + + if (obsoleteKeys.length > 0) { + const s = obsoleteKeys.length > 1 ? "s" : ""; + console.warn(` 🗑 ${obsoleteKeys.length} file${s} found.`); + // Split array in chunks of 1000 items + // (max number of entries to delete in one go is 1000 on S3) + const chunkSize = 1000; + for (let i = 0; i < obsoleteKeys.length; i += chunkSize) { + const chunk = obsoleteKeys.slice(i, i + chunkSize); + await this.fileStorageService.deleteMultipleFiles(...chunk); + } + console.warn( + ` ✅ ${obsoleteKeys.length} file${s} deleted from bucket` + ); + console.info(` → ${obsoleteKeys.join(", ")}`); + } else { + console.warn(" 🙅 No obsolete file found"); + } + } else { + console.info(" 🙅 No image found on S3 bucket"); + } + } +} diff --git a/confiture-web-app/package.json b/confiture-web-app/package.json index 1d5c66b10..1a8eea77e 100644 --- a/confiture-web-app/package.json +++ b/confiture-web-app/package.json @@ -15,21 +15,34 @@ "dependencies": { "@gouvfr/dsfr": "1.12.1", "@sentry/tracing": "^7.37.2", + "@sentry/vite-plugin": "^0.3.0", "@sentry/vue": "^7.37.2", + "@tiptap/extension-code-block-lowlight": "^2.5.9", + "@tiptap/extension-highlight": "^2.5.9", + "@tiptap/extension-image": "^2.6.6", + "@tiptap/extension-link": "^2.5.9", + "@tiptap/extension-task-item": "^2.5.9", + "@tiptap/extension-task-list": "^2.5.9", + "@tiptap/extension-typography": "^2.5.9", + "@tiptap/pm": "^2.5.9", + "@tiptap/starter-kit": "^2.5.9", + "@tiptap/vue-3": "^2.5.9", "@unhead/vue": "^1.5.3", + "@vitejs/plugin-vue": "^4.4.1", "dompurify": "^2.4.1", + "highlight.js": "^11.10.0", "jwt-decode": "^3.1.2", "ky": "^0.33.0", "lodash-es": "^4.17.21", + "lowlight": "^3.1.0", "marked": "^4.2.4", "pinia": "^2.0.28", "slugify": "^1.6.5", + "tiptap-markdown": "^0.8.10", + "vite": "^4.5.0", "vue": "^3.3.8", "vue-matomo": "^4.2.0", - "vue-router": "^4.2.5", - "vite": "^4.5.0", - "@vitejs/plugin-vue": "^4.4.1", - "@sentry/vite-plugin": "^0.3.0" + "vue-router": "^4.2.5" }, "devDependencies": { "@types/dompurify": "^2.4.0", diff --git a/confiture-web-app/src/assets/images/code-block.svg b/confiture-web-app/src/assets/images/code-block.svg new file mode 100644 index 000000000..9db393b1b --- /dev/null +++ b/confiture-web-app/src/assets/images/code-block.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/confiture-web-app/src/assets/images/strikethrough-2.svg b/confiture-web-app/src/assets/images/strikethrough-2.svg new file mode 100644 index 000000000..b96583fc6 --- /dev/null +++ b/confiture-web-app/src/assets/images/strikethrough-2.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/confiture-web-app/src/components/audit/AuditGenerationCriterium.vue b/confiture-web-app/src/components/audit/AuditGenerationCriterium.vue index f4d40356e..ac7c7db85 100644 --- a/confiture-web-app/src/components/audit/AuditGenerationCriterium.vue +++ b/confiture-web-app/src/components/audit/AuditGenerationCriterium.vue @@ -13,10 +13,12 @@ import { AuditType, CriterionResultUserImpact, CriteriumResult, - CriteriumResultStatus + CriteriumResultStatus, + FileDisplay } from "../../types"; import { formatStatus, + getUploadUrl, handleFileDeleteError, handleFileUploadError } from "../../utils"; @@ -119,12 +121,12 @@ function toggleTransverseComment() { const notify = useNotifications(); -const errorMessage: Ref = ref(null); +const errorMessage: Ref = ref(null); const criteriumNotCompliantAccordion = ref>(); function handleUploadExample(file: File) { - store + return store .uploadExampleImage( props.auditUniqueId, props.page.id, @@ -142,6 +144,28 @@ function handleUploadExample(file: File) { criteriumNotCompliantAccordion.value?.onFileRequestFinished(); }); } +function handleUploadExampleInEditor(file: File) { + return store + .uploadExampleImage( + props.auditUniqueId, + props.page.id, + props.topicNumber, + props.criterium.number, + file, + FileDisplay.EDITOR + ) + .then((response: AuditFile) => { + errorMessage.value = null; + return getUploadUrl(response.key); + }) + .catch(async (error) => { + errorMessage.value = await handleFileUploadError(error); + throw error; + }) + .finally(() => { + criteriumNotCompliantAccordion.value?.onFileRequestFinished(); + }); +} const deleteFileModalRef = ref>(); const fileToDelete = ref(); @@ -332,6 +356,7 @@ const showTransverseStatus = computed(() => { @@ -339,6 +364,7 @@ const showTransverseStatus = computed(() => { @@ -349,9 +375,14 @@ const showTransverseStatus = computed(() => { ref="criteriumNotCompliantAccordion" :comment="result.notCompliantComment" :user-impact="result.userImpact" - :example-images="result.exampleImages" + :example-images=" + result.exampleImages.filter( + (auditFile: AuditFile) => auditFile.display === FileDisplay.ATTACHMENT + ) + " :quick-win="result.quickWin" :error-message="errorMessage" + :upload-fn="handleUploadExampleInEditor" @update:comment="updateResultComment($event, 'notCompliantComment')" @update:user-impact="updateResultImpact($event)" @upload-file="handleUploadExample" diff --git a/confiture-web-app/src/components/audit/CriteriumCompliantAccordion.vue b/confiture-web-app/src/components/audit/CriteriumCompliantAccordion.vue index abdc71334..5b27a12a8 100644 --- a/confiture-web-app/src/components/audit/CriteriumCompliantAccordion.vue +++ b/confiture-web-app/src/components/audit/CriteriumCompliantAccordion.vue @@ -1,40 +1,46 @@