Merge branch 'master' into gfay_refacto_nest

.gitignore

@@ -35,4 +35,4 @@ lerna-debug.log*
 !.vscode/extensions.json
 
 .env
-contours-communes.json
+contours-communes.json

apps/legacy-api/models/base-locale.js

@@ -124,7 +124,7 @@ function parseQueryFilters(query) {
 
   if (query.email) {
     if (typeof query.email === 'string' && checkValidEmail(query.email)) {
-      filters.emails = {$eq: query.email}
+      filters.emails = {$regex: new RegExp(`^${query.email}$`, 'i')}
     } else {
       throw createHttpError(400, 'La valeur du champ "email" est invalide')
     }

apps/legacy-api/routes/__tests__/base-locale.js

@@ -95,41 +95,6 @@ test.serial('create a BaseLocale / invalid commune', async t => {
   })
 })
 
-test.serial('get all BaseLocales', async t => {
-  const _idBalA = new mongo.ObjectId()
-  const _idBalB = new mongo.ObjectId()
-
-  await mongo.db.collection('bases_locales').insertOne({
-    _id: _idBalA,
-    nom: 'foo',
-    emails: ['[email protected]'],
-    commune: '27115',
-    token: 'coucou',
-    _created: new Date('2019-01-01'),
-    _updated: new Date('2019-01-01'),
-    _deleted: null
-  })
-
-  await mongo.db.collection('bases_locales').insertOne({
-    _id: _idBalB,
-    nom: 'fobaro',
-    emails: ['[email protected]'],
-    commune: '27115',
-    token: 'hello',
-    _created: new Date('2019-01-01'),
-    _updated: new Date('2019-01-01'),
-    _deleted: new Date('2020-01-01')
-  })
-
-  const {body, status} = await request(getApp())
-    .get('/bases-locales')
-
-  t.is(status, 200)
-  t.is(body.length, 2)
-  t.true(SAFE_FIELDS.every(k => k in body[0]))
-  t.is(Object.keys(body[0]).length, SAFE_FIELDS.length)
-})
-
 test.serial('get a BaseLocal / with admin token', async t => {
   const _id = new mongo.ObjectId()
   await mongo.db.collection('bases_locales').insertOne({

apps/legacy-api/routes/__tests__/stats.js

@@ -3,7 +3,6 @@ const request = require('supertest')
 const express = require('express')
 const {MongoMemoryServer} = require('mongodb-memory-server')
 const mongo = require('../../util/mongo')
-const {prepareContoursCommunes} = require('../../util/contours-communes')
 const routes = require('../stats')
 
 function getApp() {
@@ -21,10 +20,6 @@ test.before('start server', async () => {
   await mongo.connect(mongod.getUri())
 })
 
-test.before('prepare contours communes', async () => {
-  await prepareContoursCommunes()
-})
-
 test.after.always('cleanup', async () => {
   await mongo.disconnect()
   await mongod.stop()

apps/legacy-api/routes/index.js

@@ -96,14 +96,6 @@ app.param('toponymeId', w(async (req, res, next) => {
 
 // DONE
 app.route('/bases-locales')
-  .get(w(async (req, res) => {
-    const basesLocales = await BaseLocale.fetchAll()
-    const basesLocalesExpanded = await Promise.all(
-      basesLocales.map(baseLocale => expandBalWithNumeros(baseLocale))
-    )
-
-    res.send(basesLocalesExpanded.map(bal => BaseLocale.filterSensitiveFields(bal)))
-  }))
   .post(w(async (req, res) => {
     const baseLocale = await BaseLocale.create(req.body)
     const baseLocaleExpanded = await expandBalWithNumeros(baseLocale)

apps/legacy-api/sync/index.js

@@ -1,5 +1,4 @@
 const {setTimeout} = require('timers/promises')
-const bluebird = require('bluebird')
 const createError = require('http-errors')
 const hasha = require('hasha')
 const {sub} = require('date-fns')
@@ -8,6 +7,7 @@ const {sendMail} = require('../util/sendmail')
 const createPublicationNotificationEmail = require('../emails/bal-publication-notification')
 const BaseLocale = require('../models/base-locale')
 const {fetchHabilitation} = require('../habilitations')
+const {getMongoCache, setMongoCache} = require('../tasks/mongo-cache')
 const {getCurrentRevision, publishNewRevision, getCurrentRevisions} = require('./api-depot')
 
 async function detectOutdated() {
@@ -54,7 +54,11 @@ async function updateSyncInfo(balId) {
 
   const currentRevision = await getCurrentRevision(codeCommune)
 
-  if (currentRevision._id !== baseLocale.sync.lastUploadedRevisionId) {
+  if (!currentRevision) {
+    console.error(`Comportement inattendu : pas de révision courante pour la commune ${codeCommune}`)
+  }
+
+  if (currentRevision?._id !== baseLocale.sync.lastUploadedRevisionId) {
     return applyAndReturnSync(baseLocale, {
       status: 'conflict',
       isPaused: true
@@ -199,8 +203,9 @@ async function updateConflictStatus(codeCommune) {
     return
   }
 
-  await Promise.all(basesLocales.map(async baseLocale => {
+  for (const baseLocale of basesLocales) {
     if (currentRevision._id === baseLocale.sync.lastUploadedRevisionId) {
+      /* eslint-disable-next-line no-await-in-loop */
       await mongo.db.collection('bases_locales').updateOne(
         {_id: baseLocale._id, status: 'replaced'},
         {
@@ -211,6 +216,7 @@ async function updateConflictStatus(codeCommune) {
         }
       )
     } else {
+      /* eslint-disable-next-line no-await-in-loop */
       await mongo.db.collection('bases_locales').updateOne(
         {_id: baseLocale._id, status: 'published'},
         {
@@ -221,24 +227,28 @@ async function updateConflictStatus(codeCommune) {
         }
       )
     }
-  }))
+  }
 }
 
-let detectConflictPublishedSince = new Date('2000-01-01')
-
 async function detectConflict() {
   const futurePublishedSince = new Date()
-
+  const detectConflictPublishedSince = await getMongoCache('detectConflictPublishedSince')
   const currentRevisions = await getCurrentRevisions(detectConflictPublishedSince)
   const revisedCommunes = currentRevisions.map(r => r.codeCommune)
 
-  detectConflictPublishedSince = futurePublishedSince
+  await setMongoCache('detectConflictPublishedSince', futurePublishedSince)
 
   await setTimeout(5000)
 
-  await bluebird.map(revisedCommunes, async codeCommune => {
-    await updateConflictStatus(codeCommune)
-  }, {concurrency: 10})
+  for (const codeCommune of revisedCommunes) {
+    try {
+    /* eslint-disable-next-line no-await-in-loop */
+      await updateConflictStatus(codeCommune)
+    } catch (error) {
+      console.error(`Unable to detect conflict for ${codeCommune}`)
+      console.error(error)
+    }
+  }
 }
 
 async function syncOutdated() {

apps/legacy-api/tasks/mongo-cache.js

@@ -0,0 +1,17 @@
+const mongo = require('../util/mongo')
+
+async function getMongoCache(name) {
+  const cache = await mongo.db.collection('_mongo-cache').findOne({name})
+
+  return cache?.value
+}
+
+function setMongoCache(name, value) {
+  return mongo.db.collection('_mongo-cache').updateOne(
+    {name},
+    {$set: {value}},
+    {upsert: true}
+  )
+}
+
+module.exports = {getMongoCache, setMongoCache}

apps/legacy-api/tasks/task-queue.js

@@ -0,0 +1,27 @@
+class TaskQueue {
+  constructor() {
+    this.queue = []
+    this.isTaskRunning = false
+  }
+
+  pushTask(task) {
+    this.queue.push(task)
+    if (!this.isTaskRunning) {
+      this.runTaskQueue()
+    }
+  }
+
+  async runTaskQueue() {
+    this.isTaskRunning = true
+
+    while (this.queue.length > 0) {
+      const task = this.queue.shift()
+      // eslint-disable-next-line no-await-in-loop
+      await task()
+    }
+
+    this.isTaskRunning = false
+  }
+}
+
+module.exports = {TaskQueue}

migrations/2023-10-12-init-mongo-cache.js

@@ -0,0 +1,21 @@
+#!/usr/bin/env node
+
+require('dotenv').config()
+const {sub} = require('date-fns')
+const mongo = require('../lib/util/mongo')
+const {setMongoCache} = require('../lib/tasks/mongo-cache')
+
+async function main() {
+  await mongo.connect()
+
+  const initCacheTs = sub(new Date(), {hours: 1})
+  await setMongoCache('detectConflictPublishedSince', initCacheTs)
+
+  await mongo.disconnect()
+  process.exit()
+}
+
+main().catch(error => {
+  console.error(error)
+  process.exit(1)
+})

package.json

@@ -61,6 +61,7 @@
     "class-validator": "^0.14.0",
     "compression": "^1.7.4",
     "cors": "^2.8.5",
+    "cron": "^3.1.1",
     "csv-write-stream": "^2.0.0",
     "date-fns": "^2.30.0",
     "dotenv": "^16.3.1",