fix: processor memory limit (#356)

mission-apprentissage · Jun 28, 2024 · 9e3c713 · 9e3c713
2 parents 185a197 + 0cf4dc7
commit 9e3c713
Show file tree

Hide file tree

Showing 26 changed files with 5,916 additions and 4,396 deletions.
diff --git a/.prettierignore b/.prettierignore
@@ -1,3 +1,5 @@
 .yarn
 .talismanrc
 .infra/files/configs/mongodb/seed.gpg
+.infra/files/configs/mongodb/mongo_keyfile.txt
+.infra/local/mongo_keyfile
diff --git a/.talismanrc b/.talismanrc
@@ -95,7 +95,7 @@ fileignoreconfig:
 - filename: server/.env.local.example
   checksum: b760092fca0bbc8ba8c1d1d162b0461b992de867cbfde3dc4556226a1129e484
 - filename: server/.env.test
-  checksum: 39ceb0a4f3d9cffe8d980a69205a2b800a875cbe994842f97ff1f9282b2a6e1f
+  checksum: ce67d0410bfd71366b84b5d22708befe0643e3756e2474d55552caad24fafe3c
 - filename: server/config/config.ts
   checksum: dee90e5b9ea1162aaffc204bd46a7801f8abba89fddd3f7077990321299ff3bf
 - filename: server/dev/load-env.mjs

diff --git a/Dockerfile b/Dockerfile
@@ -47,6 +47,7 @@ COPY ./server/static /app/server/static
 
 EXPOSE 5000
 WORKDIR /app/server
+ENV NODE_OPTIONS=--max_old_space_size=2048
 CMD ["node", "dist/index.js", "start"]
 
 

diff --git a/package.json b/package.json
@@ -17,10 +17,10 @@
     "shared"
   ],
   "dependencies": {
-    "dotenv": "^16.0.3",
-    "eslint-plugin-react": "^7.32.2",
+    "dotenv": "^16.4.5",
+    "eslint-plugin-react": "^7.34.2",
     "global": "4.4.0",
-    "husky": "^8.0.3"
+    "husky": "^9.0.11"
   },
   "scripts": {
     "setup": ".bin/mna-bal init:env",
@@ -62,36 +62,36 @@
     "foreach:ci": "yarn foreach:seq -p"
   },
   "devDependencies": {
-    "@commitlint/cli": "^17.7.1",
-    "@commitlint/config-conventional": "^17.7.0",
+    "@commitlint/cli": "^19.3.0",
+    "@commitlint/config-conventional": "^19.2.2",
     "@cypress/chrome-recorder": "^2.3.1",
     "@semantic-release/changelog": "^6.0.3",
     "@semantic-release/exec": "^6.0.3",
-    "@typescript-eslint/eslint-plugin": "^5.62.0",
-    "@typescript-eslint/parser": "^5.62.0",
-    "@vitest/coverage-v8": "^0.34.2",
-    "commander": "^10.0.1",
-    "cypress": "^12.17.4",
-    "eslint": "^8.47.0",
-    "eslint-config-next": "^13.4.17",
-    "eslint-import-resolver-typescript": "^3.6.0",
-    "eslint-plugin-cypress": "^2.14.0",
-    "eslint-plugin-import": "^2.28.0",
-    "eslint-plugin-jsx-a11y": "^6.7.1",
-    "eslint-plugin-mocha": "^10.1.0",
-    "eslint-plugin-n": "^16.2.0",
+    "@typescript-eslint/eslint-plugin": "^7.12.0",
+    "@typescript-eslint/parser": "^7.12.0",
+    "@vitest/coverage-v8": "^1.6.0",
+    "commander": "^12.1.0",
+    "cypress": "^13.11.0",
+    "eslint": "^8.57.0",
+    "eslint-config-next": "^13.5.6",
+    "eslint-import-resolver-typescript": "^3.6.1",
+    "eslint-plugin-cypress": "^3.3.0",
+    "eslint-plugin-import": "^2.29.1",
+    "eslint-plugin-jsx-a11y": "^6.8.0",
+    "eslint-plugin-mocha": "^10.4.3",
+    "eslint-plugin-n": "^17.8.1",
     "eslint-plugin-node": "^11.1.0",
-    "eslint-plugin-simple-import-sort": "^10.0.0",
-    "eslint-plugin-unused-imports": "^2.0.0",
+    "eslint-plugin-simple-import-sort": "^12.1.0",
+    "eslint-plugin-unused-imports": "^4.0.0",
     "eslint-plugin-zod": "^1.4.0",
-    "lint-staged": "^14.0.1",
-    "node-talisman": "^1.29.10",
-    "prettier": "^3.0.3",
-    "semantic-release": "^21.0.7",
-    "semantic-release-slack-bot": "^4.0.0",
-    "typescript": "^5.1.6",
-    "vite-tsconfig-paths": "^4.2.0",
-    "vitest": "^0.34.2"
+    "lint-staged": "^15.2.5",
+    "node-talisman": "^1.29.11",
+    "prettier": "^3.3.1",
+    "semantic-release": "^24.0.0",
+    "semantic-release-slack-bot": "^4.0.2",
+    "typescript": "^5.4.5",
+    "vite-tsconfig-paths": "^4.3.2",
+    "vitest": "^1.6.0"
   },
   "prettier": {
     "printWidth": 120,

diff --git a/server/package.json b/server/package.json
@@ -6,6 +6,7 @@
   "author": "Mission apprentissage",
   "license": "MIT",
   "type": "module",
+  "private": true,
   "engines": {
     "node": ">=20",
     "npm": "please-use-yarn"
@@ -18,12 +19,12 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@fastify/auth": "^4.4.0",
-    "@fastify/cookie": "^9.1.0",
-    "@fastify/cors": "^8.4.1",
-    "@fastify/multipart": "^8.0.0",
-    "@fastify/swagger": "^8.12.0",
-    "@fastify/swagger-ui": "^1.10.1",
+    "@fastify/auth": "^4.6.1",
+    "@fastify/cookie": "^9.3.1",
+    "@fastify/cors": "^8.5.0",
+    "@fastify/multipart": "^8.2.0",
+    "@fastify/swagger": "^8.14.0",
+    "@fastify/swagger-ui": "^1.10.2",
     "@hapi/boom": "^10.0.1",
     "@immobiliarelabs/fastify-sentry": "^7.1.1",
     "@sentry/integrations": "^7.77.0",
@@ -36,31 +37,31 @@
     "basic-ftp": "^5.0.4",
     "bunyan": "^1.8.15",
     "bunyan-prettystream": "^0.1.3",
-    "chardet": "^1.6.0",
+    "chardet": "^1.6.1",
     "checksum-stream": "^1.0.3",
-    "clamscan": "^2.1.2",
-    "commander": "^10.0.1",
+    "clamscan": "^2.2.1",
+    "commander": "^12.1.0",
     "company-email-validator": "^1.1.0",
-    "convert-csv-to-json": "^2.0.0",
+    "convert-csv-to-json": "^2.47.0",
     "cron-parser": "^4.9.0",
-    "csv-parse": "^5.4.0",
-    "csv-stringify": "^6.4.0",
+    "csv-parse": "^5.5.6",
+    "csv-stringify": "^6.5.0",
     "date-fns": "^2.30.0",
     "deep-object-diff": "^1.1.9",
     "deepmerge": "^4.3.1",
-    "ejs": "^3.1.9",
-    "env-var": "^7.3.1",
-    "fastify": "^4.21.0",
-    "fastify-type-provider-zod": "^1.1.9",
+    "ejs": "^3.1.10",
+    "env-var": "^7.5.0",
+    "fastify": "^4.27.0",
+    "fastify-type-provider-zod": "^1.2.0",
     "iconv-lite": "^0.6.3",
-    "job-processor": "^1.4.12",
-    "jsonwebtoken": "^9.0.1",
+    "job-processor": "^1.5.0",
+    "jsonwebtoken": "^9.0.2",
     "lil-http-terminator": "^1.2.3",
     "lodash-es": "^4.17.21",
-    "migrate-mongo": "^10.0.0",
-    "mjml": "^4.14.1",
-    "mongodb": "^5.7.0",
-    "nodemailer": "^6.9.4",
+    "migrate-mongo": "^11.0.0",
+    "mjml": "^4.15.3",
+    "mongodb": "^6.7.0",
+    "nodemailer": "^6.9.13",
     "nodemailer-html-to-text": "^3.2.0",
     "oleoduc": "^0.8.0",
     "omit-deep": "^0.3.0",
@@ -69,31 +70,31 @@
     "rate-limiter-flexible": "^2.4.2",
     "shared": "workspace:*",
     "tcp-port-used": "^1.0.2",
-    "uuid": "^9.0.0",
+    "uuid": "^9.0.1",
     "xlsx": "^0.18.5",
-    "zod": "^3.22.4",
-    "zod-mongodb-schema": "^1.0.0"
+    "zod": "^3.23.8",
+    "zod-mongodb-schema": "^1.0.2"
   },
   "devDependencies": {
-    "@types/bunyan": "^1.8.8",
-    "@types/bunyan-prettystream": "^0.1.32",
-    "@types/clamscan": "^2.0.4",
-    "@types/ejs": "^3.1.2",
-    "@types/jsonwebtoken": "^9.0.2",
-    "@types/lodash-es": "^4.17.8",
-    "@types/migrate-mongo": "^10.0.0",
-    "@types/mjml": "^4.7.1",
-    "@types/nodemailer": "^6.4.9",
-    "@types/nodemailer-html-to-text": "^3.1.0",
-    "@types/omit-deep": "^0.3.0",
-    "@types/tcp-port-used": "^1.0.1",
-    "@types/uuid": "^9.0.2",
-    "dotenv": "^16.3.1",
+    "@types/bunyan": "^1.8.11",
+    "@types/bunyan-prettystream": "^0.1.35",
+    "@types/clamscan": "^2.0.8",
+    "@types/ejs": "^3.1.5",
+    "@types/jsonwebtoken": "^9.0.6",
+    "@types/lodash-es": "^4.17.12",
+    "@types/migrate-mongo": "^10.0.4",
+    "@types/mjml": "^4.7.4",
+    "@types/nodemailer": "^6.4.15",
+    "@types/nodemailer-html-to-text": "^3.1.3",
+    "@types/omit-deep": "^0.3.2",
+    "@types/tcp-port-used": "^1.0.4",
+    "@types/uuid": "^9.0.8",
+    "dotenv": "^16.4.5",
     "json-schema": "^0.4.0",
-    "nock": "^13.3.3",
-    "tsup": "^7.2.0",
-    "typescript": "^5.1.6",
-    "vitest": "^0.34.6"
+    "nock": "^13.5.4",
+    "tsup": "^7.3.0",
+    "typescript": "^5.4.5",
+    "vitest": "^1.6.0"
   },
   "files": [
     "src/**/*",

diff --git a/server/src/common/apis/deca.ts b/server/src/common/apis/deca.ts
@@ -47,6 +47,7 @@ export const getDeca = async (
   page: number,
   product: "LBA" | "TDB" = "LBA"
 ): Promise<ApiDeca> => {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
   return executeWithRateLimiting(async (client: any) => {
     // axiosRetry(client, { retries: 3 });
 
@@ -72,13 +73,15 @@ export const getDeca = async (
       const duration = formatDuration(intervalToDuration({ start: startDate, end: endDate })) || `${ts}ms`;
       console.log(duration);
       logger.debug(
-        `[API Deca] Récupération contrats du ${dateDebut} au ${dateFin} - page ${page} sur ${response?.data?.metadonnees
-          ?.totalPages} ${response.cached ? "(depuis le cache)" : ""}`
+        `[API Deca] Récupération contrats du ${dateDebut} au ${dateFin} - page ${page} sur ${
+          response?.data?.metadonnees?.totalPages
+        } ${response.cached ? "(depuis le cache)" : ""}`
       );
       if (!response?.data) {
         throw new ApiError("Api Deca", "No data received");
       }
       return response.data;
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
     } catch (e: any) {
       logger.info(e.response);
       if (!e.response) logger.info(e);

diff --git a/server/src/common/logger.ts b/server/src/common/logger.ts
@@ -33,6 +33,7 @@ const createStreams = () => {
   return streams;
 };
 
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
 function errorSerialiser(err: any) {
   return {
     ...bunyan.stdSerializers.err(err),

diff --git a/server/src/common/services/sentry/sentry.ts b/server/src/common/services/sentry/sentry.ts
@@ -1,5 +1,4 @@
 import fastifySentryPlugin, { FastifySentryOptions } from "@immobiliarelabs/fastify-sentry";
-import { CaptureConsole, ExtraErrorData } from "@sentry/integrations";
 import * as Sentry from "@sentry/node";
 import { FastifyRequest } from "fastify";
 
@@ -14,10 +13,9 @@ function getOptions(): Sentry.NodeOptions {
     release: config.version,
     enabled: config.env !== "local",
     integrations: [
-      new Sentry.Integrations.Http({ tracing: true }),
-      new Sentry.Integrations.Mongo({ useMongoose: false }),
-      new CaptureConsole({ levels: ["error"] }) as any,
-      new ExtraErrorData({ depth: 16 }) as any,
+      Sentry.httpIntegration({}),
+      Sentry.captureConsoleIntegration({ levels: ["error"] }),
+      Sentry.extraErrorDataIntegration({ depth: 16 }),
     ],
   };
 }
@@ -87,26 +85,3 @@ export function initSentryFastify(app: Server) {
   // @ts-expect-error
   app.register(fastifySentryPlugin, { options, ...getOptions() });
 }
-
-function getTransation() {
-  return Sentry.getCurrentHub()?.getScope()?.getSpan();
-}
-
-export function startSentryPerfRecording(
-  category: string,
-  operation: string,
-  data: {
-    [key: string]: unknown;
-  } = {}
-): () => void {
-  const childTransaction =
-    getTransation()?.startChild({
-      op: category,
-      description: operation,
-      data,
-    }) ?? null;
-
-  return () => {
-    childTransaction?.finish();
-  };
-}
diff --git a/server/src/common/utils/mongodbUtils.ts b/server/src/common/utils/mongodbUtils.ts
@@ -1,4 +1,5 @@
 import { captureException } from "@sentry/node";
+import { isEqual } from "lodash-es";
 import { Collection, CollectionInfo, MongoClient, MongoServerError } from "mongodb";
 import { CollectionName, IModelDescriptor } from "shared/models/common";
 import { IDocumentMap, modelDescriptors } from "shared/models/models";
@@ -156,17 +157,55 @@ export const createIndexes = async () => {
     if (!descriptor.indexes) {
       return;
     }
+    const indexes = await getDbCollection(descriptor.collectionName)
+      .listIndexes()
+      .toArray()
+      .catch((err) => {
+        // NamespaceNotFound
+        if (err.code === 26) {
+          return [];
+        }
+        throw err;
+      });
+    const indexesToRemove = new Set(indexes.filter((i) => i.name !== "_id_"));
+
     logger.info(`Create indexes for collection ${descriptor.collectionName}`);
     await Promise.all(
-      descriptor.indexes.map(async ([index, options]) => {
+      descriptor.indexes.map(async ([index, options]): Promise<void> => {
         try {
-          await getDbCollection(descriptor.collectionName).createIndex(index, options);
+          const existingIndex =
+            // Use Object.entries because order matters
+            indexes.find((i) => isEqual(Object.entries(i.key), Object.entries(index)) || options.name === i.name) ??
+            null;
+
+          if (existingIndex) {
+            indexesToRemove.delete(existingIndex);
+          }
+
+          await getDbCollection(descriptor.collectionName)
+            .createIndex(index, options)
+            .catch(async (err) => {
+              // IndexOptionsConflict & IndexKeySpecsConflict
+              if (err.code === 85 || err.code === 86) {
+                await getDbCollection(descriptor.collectionName).dropIndex(existingIndex.name);
+                await getDbCollection(descriptor.collectionName).createIndex(index, options);
+              } else {
+                throw err;
+              }
+            });
         } catch (err) {
           captureException(err);
           logger.error(`Error creating indexes for ${descriptor.collectionName}: ${err}`);
         }
       })
     );
+
+    if (indexesToRemove.size > 0) {
+      logger.warn(`Dropping extra indexes for collection ${descriptor.collectionName}`, indexesToRemove);
+      await Promise.all(
+        Array.from(indexesToRemove).map((index) => getDbCollection(descriptor.collectionName).dropIndex(index.name))
+      );
+    }
   }
 };
 

diff --git a/server/src/db/migrations/20231114155041-job_processor.ts b/server/src/db/migrations/20231114155041-job_processor.ts
@@ -15,6 +15,7 @@ export const up = async (db: Db, _client: MongoClient) => {
   ]);
 
   for await (const uploadDocumentWitJob of uploadDocumentWitJobCursor) {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const job = uploadDocumentWitJob.jobs.filter((j: any) => j.name === "import:document")[0] ?? null;
     const update: Partial<IUploadDocument> = {
       kind: "upload",
@@ -84,6 +85,7 @@ export const up = async (db: Db, _client: MongoClient) => {
   ]);
 
   for await (const doc of mailingListDocumentWittJobAndMailingListCursor) {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const job = doc.jobs.filter((j: any) => j.name === "generate:mailing-list")[0] ?? null;
     const update: Partial<IMailingListDocument> = {
       kind: "mailingList",