From 4a5d09ae9c73fc15bc5e3b6798d67924b3f0236e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Guillois?= Date: Wed, 19 Jun 2024 16:39:21 +0200 Subject: [PATCH 01/25] chore: optimize actions triggers --- .github/workflows/codeql-analysis.yml | 12 ++++++++++++ .github/workflows/github-actions.yml | 9 ++++++++- .github/workflows/review-app.yml | 9 +++++++-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index da462872e..ee972b41a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -14,9 +14,21 @@ name: "CodeQL" on: push: branches: [ main ] + paths: + - frontend/** + - queue/** + - server/** + - shared/** + - packages/** pull_request: # The branches below must be a subset of the branches above branches: [ main ] + paths: + - frontend/** + - queue/** + - server/** + - shared/** + - packages/** schedule: - cron: '21 21 * * 4' diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index bb2f6aad3..86671da59 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -1,6 +1,13 @@ name: Node.js CI -on: [push] +on: + push: + paths: + - frontend/** + - queue/** + - server/** + - shared/** + - packages/** jobs: build: diff --git a/.github/workflows/review-app.yml b/.github/workflows/review-app.yml index 5091b7235..b36ddd1b8 100644 --- a/.github/workflows/review-app.yml +++ b/.github/workflows/review-app.yml @@ -6,7 +6,12 @@ on: pull_request_target: types: [opened, closed, synchronize, reopened] branches: [main] - + paths: + - frontend/** + - queue/** + - server/** + - shared/** + - packages/** env: CLEVER_SECRET: ${{ secrets.CLEVER_SECRET }} CLEVER_TOKEN: ${{ secrets.CLEVER_TOKEN }} @@ -29,7 +34,7 @@ jobs: run: clever addon create redis-addon $PR_NAME-redis --org $ORGA_ID --plan s_mono --region par deploy-api: - if: github.event.action == 'opened' || github.event.action == 'reopened' + if: (github.event.action == 'opened' || github.event.action == 'reopened') && !github.event.pull_request.draft needs: [deploy-addons] runs-on: ubuntu-latest environment: From 1727b93ce7adab225b223add38e516bf689dc9c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Guillois?= Date: Wed, 19 Jun 2024 16:39:21 +0200 Subject: [PATCH 02/25] chore: optimize actions triggers --- .github/workflows/codeql-analysis.yml | 12 ++++++++++++ .github/workflows/github-actions.yml | 9 ++++++++- .github/workflows/review-app.yml | 9 +++++++-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index da462872e..ee972b41a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -14,9 +14,21 @@ name: "CodeQL" on: push: branches: [ main ] + paths: + - frontend/** + - queue/** + - server/** + - shared/** + - packages/** pull_request: # The branches below must be a subset of the branches above branches: [ main ] + paths: + - frontend/** + - queue/** + - server/** + - shared/** + - packages/** schedule: - cron: '21 21 * * 4' diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index bb2f6aad3..86671da59 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -1,6 +1,13 @@ name: Node.js CI -on: [push] +on: + push: + paths: + - frontend/** + - queue/** + - server/** + - shared/** + - packages/** jobs: build: diff --git a/.github/workflows/review-app.yml b/.github/workflows/review-app.yml index 5091b7235..a95271c10 100644 --- a/.github/workflows/review-app.yml +++ b/.github/workflows/review-app.yml @@ -6,7 +6,12 @@ on: pull_request_target: types: [opened, closed, synchronize, reopened] branches: [main] - + paths: + - frontend/** + - queue/** + - server/** + - shared/** + - packages/** env: CLEVER_SECRET: ${{ secrets.CLEVER_SECRET }} CLEVER_TOKEN: ${{ secrets.CLEVER_TOKEN }} @@ -29,7 +34,7 @@ jobs: run: clever addon create redis-addon $PR_NAME-redis --org $ORGA_ID --plan s_mono --region par deploy-api: - if: github.event.action == 'opened' || github.event.action == 'reopened' + if: (github.event.action == 'opened' || github.event.action == 'reopened') needs: [deploy-addons] runs-on: ubuntu-latest environment: From 0f6a6648bc538fe181f2b717c2f4bd8ef3c1f92f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Guillois?= Date: Thu, 27 Jun 2024 15:34:08 +0200 Subject: [PATCH 03/25] chore: Update react-dsfr to last version: 1.9.20 --- frontend/package.json | 2 +- yarn.lock | 17 ++--------------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index b803e93a5..4876402e3 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -21,7 +21,7 @@ ] }, "dependencies": { - "@codegouvfr/react-dsfr": "1.9.17", + "@codegouvfr/react-dsfr": "^1.9.20", "@emotion/react": "^11.11.4", "@emotion/styled": "^11.11.5", "@faker-js/faker": "^8.4.1", diff --git a/yarn.lock b/yarn.lock index b39bb8f06..063655557 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2410,20 +2410,7 @@ __metadata: languageName: node linkType: hard -"@codegouvfr/react-dsfr@npm:1.9.17": - version: 1.9.17 - resolution: "@codegouvfr/react-dsfr@npm:1.9.17" - dependencies: - tsafe: "npm:^1.6.3" - yargs-parser: "npm:^21.1.1" - bin: - copy-dsfr-to-public: bin/copy-dsfr-to-public.js - only-include-used-icons: bin/only-include-used-icons.js - checksum: 10c0/a3b5a56e2e17d69e4de635ddf8df4c0efebb854ac3524171743681064bdb8fa73522d5cd77ade5de58728871f53864c5d74478914f81c53ed012babbe0036e45 - languageName: node - linkType: hard - -"@codegouvfr/react-dsfr@npm:^1.9.11": +"@codegouvfr/react-dsfr@npm:^1.9.11, @codegouvfr/react-dsfr@npm:^1.9.20": version: 1.9.20 resolution: "@codegouvfr/react-dsfr@npm:1.9.20" dependencies: @@ -8795,7 +8782,7 @@ __metadata: version: 0.0.0-use.local resolution: "@zerologementvacant/front@workspace:frontend" dependencies: - "@codegouvfr/react-dsfr": "npm:1.9.17" + "@codegouvfr/react-dsfr": "npm:^1.9.20" "@craco/craco": "npm:^7.1.0" "@emotion/react": "npm:^11.11.4" "@emotion/styled": "npm:^11.11.5" From 0bacbc85851b0b860838d8a804f7985f98e5ec9f Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Wed, 10 Jul 2024 09:42:34 +0200 Subject: [PATCH 04/25] test(e2e): remove unused actions --- e2e/cypress/e2e/campaign.cy.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/e2e/cypress/e2e/campaign.cy.ts b/e2e/cypress/e2e/campaign.cy.ts index 027ba4585..9aa4dd8ef 100644 --- a/e2e/cypress/e2e/campaign.cy.ts +++ b/e2e/cypress/e2e/campaign.cy.ts @@ -6,10 +6,7 @@ describe('Campaign', () => { ); cy.logIn(); - cy.get('button').contains('Bâtiment/DPE').click(); - cy.get('label').contains('Nombre de logements').next().click(); - cy.wait('@findHousings'); - cy.wait('@countHousings'); + cy.wait(['@findHousings', '@countHousings']); cy.get('tbody') .find('fieldset') .then((checkboxes) => checkboxes.slice(0, 3)) @@ -69,7 +66,7 @@ describe('Campaign', () => { cy.get('div[aria-labelledby="draft-body-label"]') .type('Madame, Monsieur,{enter}') .type( - 'Marseille BB fait partie des lauréats du plan national de lutte contre les logements vacants du Ministère de la Transition écologique et de la Cohésion des Territoires. Ce plan a pour objectif d’accélérer, dans les territoires pilotes, la remise sur le marché immobilier (rénovation, location, vente, restructuration) du plus grand nombre possible de logements vacants. Dans ce cadre, l’ADIL 35 a été missionnée par Rennes Métropole pour assurer une mission d’information, de sensibilisation et d’accompagnement des propriétaires de logements vacants qui le souhaitent.{enter}' + 'Marseille BB fait partie des lauréats du plan national de lutte contre les logements vacants du Ministère de la Transition écologique et de la Cohésion des Territoires. Ce plan a pour objectif d’accélérer, dans les territoires pilotes, la remise sur le marché immobilier (rénovation, location, vente, restructuration) du plus grand nombre possible de logements vacants. Dans ce cadre, l’ADIL 13 a été missionnée par Marseille BB pour assurer une mission d’information, de sensibilisation et d’accompagnement des propriétaires de logements vacants qui le souhaitent.{enter}' ) .type( 'Un formulaire vous est proposé dans le cadre d’une enquête destinée à mieux comprendre les raisons de la vacance et s’inscrit dans une politique plus globale afin de construire l’aide qui vous sera la plus adaptée, et permettrait la remise des biens sur un marché en forte demande.{enter}' @@ -78,7 +75,7 @@ describe('Campaign', () => { 'Depuis 2011, 960 propriétaires ont été accompagnés dans le cadre de l’opération Rennes Centre Ancien. Ce sont ainsi 300 logements vacants qui ont pu être réoccupés grâce à des aides publiques.{enter}' ) .type( - 'Votre logement situé au 123 rue bidon, à Marseille a été recensé comme vacant, c’est-à-dire qu’il aurait été déclaré comme inoccupé depuis « Nombre année vacance du logement » au 1er Janvier 2022. Si tel n’est pas le cas, votre retour permettra d’actualiser l’état réel de son occupation.{enter}' + 'Votre logement situé au 123 rue bidon, à Marseille BB a été recensé comme vacant, c’est-à-dire qu’il aurait été déclaré comme inoccupé depuis « Nombre année vacance du logement » au 1er Janvier 2022. Si tel n’est pas le cas, votre retour permettra d’actualiser l’état réel de son occupation.{enter}' ) .type( 'Un formulaire vous est proposé dans le cadre d’une enquête destinée à mieux comprendre les raisons de la vacance et s’inscrit dans une politique plus globale afin de construire l’aide qui vous sera la plus adaptée, et permettrait la remise des biens sur un marché en forte demande.{enter}' From e37dad61cdc8716addf078c6c0a48408131fe502 Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Wed, 10 Jul 2024 10:15:57 +0200 Subject: [PATCH 05/25] ci(deploy): add e2e env vars to review apps --- .github/workflows/review-app.yml | 5 +++++ .talismanrc | 7 ++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/review-app.yml b/.github/workflows/review-app.yml index 2a84e6eed..c2126f2c6 100644 --- a/.github/workflows/review-app.yml +++ b/.github/workflows/review-app.yml @@ -63,6 +63,9 @@ jobs: clever env -a $APP_ALIAS | grep -v '^#' | tr -d '"' >> "$GITHUB_ENV" - name: Set environment variables + env: + E2E_EMAIL: ${{ secrets.E2E_EMAIL }} + E2E_PASSWORD: ${{ secrets.E2E_PASSWORD }} run: | clever env set AUTH_SECRET "secret" -a $APP_ALIAS clever env set CC_HEALTH_CHECK_PATH "/" -a $APP_ALIAS @@ -73,6 +76,8 @@ jobs: clever env set CEREMA_TOKEN "unused" -a $APP_ALIAS clever env set DATABASE_ENV "development" -a $APP_ALIAS clever env set DATABASE_URL $POSTGRESQL_ADDON_URI -a $APP_ALIAS + clever env set E2E_EMAIL $E2E_EMAIL -a $APP_ALIAS + clever env set E2E_PASSWORD: $E2E_PASSWORD -a $APP_ALIAS clever env set HOST "https://$APP_ALIAS.cleverapps.io" -a $APP_ALIAS clever env set METABASE_TOKEN "unused" -a $APP_ALIAS clever env set PORT "8080" -a $APP_ALIAS diff --git a/.talismanrc b/.talismanrc index 941d8bc7f..d0f9a0dea 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,7 +1,8 @@ fileignoreconfig: -- filename: .github/workflows/deploy.yml - checksum: fe575a99e7ab0a3421ca14fcff7d4997cf2bf31ddf5619b7711233f5a4a316d3 -- filename: .github/workflows/e2e.yml +- filename: .github/workflows/review-app.yml + checksum: 78d9abc578bda1e6622a67acfd199ffba381ad98cfeeb5b31bb1fe74c95fc3c1 +version: "" +hub/workflows/e2e.yml checksum: 1994a562d1d57521c10078640cd49fdccf7b3baecbaf46f3ed383ada996b5c90 - filename: .github/workflows/github-actions.yml checksum: b1a53b557b6b2ac4c57f6afce9e902582f5589f97762a963da587496a455c0c1 From 830c70976f1eb2f21bb76d3ff02a4fef61ba4d95 Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Mon, 24 Jun 2024 11:13:23 +0200 Subject: [PATCH 06/25] feat(queue): install @bull-board --- queue/package.json | 3 +++ queue/src/dashboard.ts | 26 +++++++++++++++++++++++ queue/src/server.ts | 13 +++++++----- yarn.lock | 48 ++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 83 insertions(+), 7 deletions(-) create mode 100644 queue/src/dashboard.ts diff --git a/queue/package.json b/queue/package.json index 16eb16e91..046afd505 100644 --- a/queue/package.json +++ b/queue/package.json @@ -14,6 +14,9 @@ "@aws-sdk/client-s3": "^3.577.0", "@aws-sdk/lib-storage": "^3.578.0", "@aws-sdk/s3-request-presigner": "^3.577.0", + "@bull-board/api": "5.20.2", + "@bull-board/express": "5.20.2", + "@bull-board/ui": "5.20.2", "@godaddy/terminus": "^4.12.1", "@sentry/integrations": "^7.116.0", "@sentry/node": "^7.116.0", diff --git a/queue/src/dashboard.ts b/queue/src/dashboard.ts new file mode 100644 index 000000000..f5d3d342a --- /dev/null +++ b/queue/src/dashboard.ts @@ -0,0 +1,26 @@ +import { ExpressAdapter } from '@bull-board/express'; +import { createBullBoard } from '@bull-board/api'; +import { BullMQAdapter } from '@bull-board/api/bullMQAdapter'; + +import { Queue } from 'bullmq'; +import { parseRedisUrl } from 'parse-redis-url-simple'; + +import { JOBS } from './jobs'; +import config from './config'; + +export const expressAdapter = new ExpressAdapter(); +expressAdapter.setBasePath('/admin/queues'); + +const [redis] = parseRedisUrl(config.redis.url); + +const queues = JOBS.map( + (job) => + new Queue(job, { + connection: redis + }) +).map((queue) => new BullMQAdapter(queue)); + +createBullBoard({ + queues, + serverAdapter: expressAdapter +}); diff --git a/queue/src/server.ts b/queue/src/server.ts index 0b446705d..04640349e 100644 --- a/queue/src/server.ts +++ b/queue/src/server.ts @@ -5,10 +5,11 @@ import { healthcheck, postgresCheck, redisCheck, - s3Check, + s3Check } from '@zerologementvacant/healthcheck'; import config from './config'; import { createLogger } from './logger'; +import { expressAdapter } from './dashboard'; function createServer() { const app = express(); @@ -22,12 +23,14 @@ function createServer() { checks: [ redisCheck(config.redis.url), postgresCheck(config.db.url), - s3Check(config.s3), + s3Check(config.s3) ], - logger, - }), + logger + }) ); + app.use('/admin/queues', expressAdapter.getRouter()); + async function start(): Promise { const listen = util.promisify((port: number, cb: () => void) => { return app.listen(port, cb); @@ -39,7 +42,7 @@ function createServer() { return { app, - start, + start }; } diff --git a/yarn.lock b/yarn.lock index 3c5256d32..d86dc1311 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2392,6 +2392,38 @@ __metadata: languageName: node linkType: hard +"@bull-board/api@npm:5.20.2": + version: 5.20.2 + resolution: "@bull-board/api@npm:5.20.2" + dependencies: + redis-info: "npm:^3.0.8" + peerDependencies: + "@bull-board/ui": 5.20.2 + checksum: 10c0/7bd8be3f776ac7d3adf47ba4a4b19aeb3eb32ab951f9e8212f793b3f2d3d0498c521fdf4a538851f596402c3b399ea982c2b8e6bcb01262952e9c976d5f13a96 + languageName: node + linkType: hard + +"@bull-board/express@npm:5.20.2": + version: 5.20.2 + resolution: "@bull-board/express@npm:5.20.2" + dependencies: + "@bull-board/api": "npm:5.20.2" + "@bull-board/ui": "npm:5.20.2" + ejs: "npm:^3.1.10" + express: "npm:^4.19.2" + checksum: 10c0/3e797f2f17ed091a58e7ed9d3d2cdb7d9fd0a9a55926535cf13eabb7b907d612301406e87e333145a3161d86882e3afbeb243c467ac20f3a738f44d8249d7924 + languageName: node + linkType: hard + +"@bull-board/ui@npm:5.20.2": + version: 5.20.2 + resolution: "@bull-board/ui@npm:5.20.2" + dependencies: + "@bull-board/api": "npm:5.20.2" + checksum: 10c0/9d1ec87d3de306737b5dff66bbfd8920e0d20fba0985466905ae4f3c30ed714de95cbb4757119846ba6cd46edd3147968163ea936805e855116aaded2ad98aa3 + languageName: node + linkType: hard + "@bundled-es-modules/cookie@npm:^2.0.0": version: 2.0.0 resolution: "@bundled-es-modules/cookie@npm:2.0.0" @@ -9151,6 +9183,9 @@ __metadata: "@aws-sdk/client-s3": "npm:^3.577.0" "@aws-sdk/lib-storage": "npm:^3.578.0" "@aws-sdk/s3-request-presigner": "npm:^3.577.0" + "@bull-board/api": "npm:5.20.2" + "@bull-board/express": "npm:5.20.2" + "@bull-board/ui": "npm:5.20.2" "@faker-js/faker": "npm:^8.4.1" "@godaddy/terminus": "npm:^4.12.1" "@sentry/integrations": "npm:^7.116.0" @@ -13096,7 +13131,7 @@ __metadata: languageName: node linkType: hard -"ejs@npm:^3.1.6": +"ejs@npm:^3.1.10, ejs@npm:^3.1.6": version: 3.1.10 resolution: "ejs@npm:3.1.10" dependencies: @@ -18715,7 +18750,7 @@ __metadata: languageName: node linkType: hard -"lodash@npm:^4.17.15, lodash@npm:^4.17.20, lodash@npm:^4.17.21, lodash@npm:^4.7.0": +"lodash@npm:^4.17.11, lodash@npm:^4.17.15, lodash@npm:^4.17.20, lodash@npm:^4.17.21, lodash@npm:^4.7.0": version: 4.17.21 resolution: "lodash@npm:4.17.21" checksum: 10c0/d8cbea072bb08655bb4c989da418994b073a608dffa608b09ac04b43a791b12aeae7cd7ad919aa4c925f33b48490b5cfe6c1f71d827956071dae2e7bb3a6b74c @@ -22971,6 +23006,15 @@ __metadata: languageName: node linkType: hard +"redis-info@npm:^3.0.8": + version: 3.1.0 + resolution: "redis-info@npm:3.1.0" + dependencies: + lodash: "npm:^4.17.11" + checksum: 10c0/ec0f31d97893c5828cec7166486d74198c92160c60073b6f2fe805cdf575a10ddcccc7641737d44b8f451355f0ab5b6c7b0d79e8fc24742b75dd625f91ffee38 + languageName: node + linkType: hard + "redis-parser@npm:^3.0.0": version: 3.0.0 resolution: "redis-parser@npm:3.0.0" From c152e208bdcb8fdf94021abc796c3a89af42bd09 Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Wed, 26 Jun 2024 18:29:06 +0200 Subject: [PATCH 07/25] refactor(queue): move the dashboard URL to /queues --- queue/src/dashboard.ts | 6 ++++-- queue/src/server.ts | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/queue/src/dashboard.ts b/queue/src/dashboard.ts index f5d3d342a..a3af1b235 100644 --- a/queue/src/dashboard.ts +++ b/queue/src/dashboard.ts @@ -8,8 +8,8 @@ import { parseRedisUrl } from 'parse-redis-url-simple'; import { JOBS } from './jobs'; import config from './config'; -export const expressAdapter = new ExpressAdapter(); -expressAdapter.setBasePath('/admin/queues'); +const expressAdapter = new ExpressAdapter(); +expressAdapter.setBasePath('/queues'); const [redis] = parseRedisUrl(config.redis.url); @@ -24,3 +24,5 @@ createBullBoard({ queues, serverAdapter: expressAdapter }); + +export const createDashboard = () => expressAdapter.getRouter(); diff --git a/queue/src/server.ts b/queue/src/server.ts index 04640349e..13aa64db3 100644 --- a/queue/src/server.ts +++ b/queue/src/server.ts @@ -9,7 +9,7 @@ import { } from '@zerologementvacant/healthcheck'; import config from './config'; import { createLogger } from './logger'; -import { expressAdapter } from './dashboard'; +import { createDashboard } from './dashboard'; function createServer() { const app = express(); @@ -29,7 +29,7 @@ function createServer() { }) ); - app.use('/admin/queues', expressAdapter.getRouter()); + app.use('/queues', createDashboard()); async function start(): Promise { const listen = util.promisify((port: number, cb: () => void) => { From 8c0aa50203faf7f1cce05e1bc17e1ac8b70e5477 Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Wed, 10 Jul 2024 14:49:38 +0200 Subject: [PATCH 08/25] feat(queue): add basic auth to the queue dashboard --- .talismanrc | 2 ++ queue/package.json | 2 ++ queue/src/.htpasswd | 1 + queue/src/basic-auth.ts | 8 +++++++ queue/src/server.ts | 3 ++- yarn.lock | 46 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 queue/src/.htpasswd create mode 100644 queue/src/basic-auth.ts diff --git a/.talismanrc b/.talismanrc index d0f9a0dea..1f9ef3491 100644 --- a/.talismanrc +++ b/.talismanrc @@ -79,6 +79,8 @@ hub/workflows/e2e.yml checksum: df71affe33700cb0a36ee0b4824ef60e7d8e5a08f3d4d16ca53c31e9085b45ae - filename: queue/README.md checksum: fe2ac5277a42e6bf8dfc1e0fe53cc3308bba9019d9091f4b8c291e4a2e4a4eb8 +- filename: queue/src/.htpasswd + checksum: 7670652d46bed42247610009a41eadea1d18ff839db9fa831288cb6ab19792e1 - filename: queue/src/config.ts checksum: 8ca4d9a0b6118457fb7dcea24e43645eb63bea92d9e0f1d0c4ae763ea55491a5 - filename: queue/src/events/index.ts diff --git a/queue/package.json b/queue/package.json index 046afd505..9407552cf 100644 --- a/queue/package.json +++ b/queue/package.json @@ -32,6 +32,7 @@ "dotenv": "^16.4.5", "exceljs": "^4.4.0", "express": "^4.19.2", + "http-auth": "^4.2.0", "parse-redis-url-simple": "^1.0.2" }, "devDependencies": { @@ -41,6 +42,7 @@ "@types/async": "^3.2.24", "@types/convict": "^6.1.6", "@types/express": "^4.17.21", + "@types/http-auth": "^4", "@types/jest": "^29.5.12", "@types/node": "^20.12.12", "@types/supertest": "^6.0.2", diff --git a/queue/src/.htpasswd b/queue/src/.htpasswd new file mode 100644 index 000000000..0daeb4f9f --- /dev/null +++ b/queue/src/.htpasswd @@ -0,0 +1 @@ +zlv@beta.gouv.fr:$apr1$6ts5byl3$575IQxAARDD6llSdjDpQM0 diff --git a/queue/src/basic-auth.ts b/queue/src/basic-auth.ts new file mode 100644 index 000000000..64f1c836d --- /dev/null +++ b/queue/src/basic-auth.ts @@ -0,0 +1,8 @@ +import auth from 'http-auth'; +import path from 'node:path'; + +export const createBasicAuth = () => + auth.basic({ + file: path.join(__dirname, '.htpasswd'), + skipUser: true + }); diff --git a/queue/src/server.ts b/queue/src/server.ts index 13aa64db3..48f67d20e 100644 --- a/queue/src/server.ts +++ b/queue/src/server.ts @@ -10,6 +10,7 @@ import { import config from './config'; import { createLogger } from './logger'; import { createDashboard } from './dashboard'; +import { createBasicAuth } from './basic-auth'; function createServer() { const app = express(); @@ -29,7 +30,7 @@ function createServer() { }) ); - app.use('/queues', createDashboard()); + app.use('/queues', createBasicAuth().check(createDashboard())); async function start(): Promise { const listen = util.promisify((port: number, cb: () => void) => { diff --git a/yarn.lock b/yarn.lock index d86dc1311..c2a2353ce 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7955,6 +7955,15 @@ __metadata: languageName: node linkType: hard +"@types/http-auth@npm:^4": + version: 4.1.4 + resolution: "@types/http-auth@npm:4.1.4" + dependencies: + "@types/node": "npm:*" + checksum: 10c0/24a9497ef835c87b3df2d74fb6fd28960ce2921c944a20dbc5ba93e46561626d7381c8272659af930925f5149fe95f7515f892f5a9966ff89c91ffd6bdc8db02 + languageName: node + linkType: hard + "@types/http-errors@npm:*": version: 2.0.4 resolution: "@types/http-errors@npm:2.0.4" @@ -9195,6 +9204,7 @@ __metadata: "@types/async": "npm:^3.2.24" "@types/convict": "npm:^6.1.6" "@types/express": "npm:^4.17.21" + "@types/http-auth": "npm:^4" "@types/jest": "npm:^29.5.12" "@types/node": "npm:^20.12.12" "@types/supertest": "npm:^6.0.2" @@ -9210,6 +9220,7 @@ __metadata: dotenv: "npm:^16.4.5" exceljs: "npm:^4.4.0" express: "npm:^4.19.2" + http-auth: "npm:^4.2.0" jest: "npm:^29.7.0" jest-extended: "npm:^4.0.2" nodemon: "npm:^3.1.0" @@ -9709,6 +9720,22 @@ __metadata: languageName: node linkType: hard +"apache-crypt@npm:^1.1.2": + version: 1.2.6 + resolution: "apache-crypt@npm:1.2.6" + dependencies: + unix-crypt-td-js: "npm:^1.1.4" + checksum: 10c0/00ce45e671f256f3bbdcd47da57b9d9007af3e70102316304bfb1fb4f28610ea9b733e616a90079ee7c2bf1adebdda3fd99cafe2ac668ad55b0323c79df64f67 + languageName: node + linkType: hard + +"apache-md5@npm:^1.0.6": + version: 1.1.8 + resolution: "apache-md5@npm:1.1.8" + checksum: 10c0/423aa1baddcedc42e2fdf52efcf7fae2e7de9535e6ca7dd4a049f49fb5ec9b6a4469f327e02268088ed3dacdbec6f1ea4132941e2d75899c4e412421e6ffcbfc + languageName: node + linkType: hard + "append-field@npm:^1.0.0": version: 1.0.0 resolution: "append-field@npm:1.0.0" @@ -15729,6 +15756,18 @@ __metadata: languageName: node linkType: hard +"http-auth@npm:^4.2.0": + version: 4.2.0 + resolution: "http-auth@npm:4.2.0" + dependencies: + apache-crypt: "npm:^1.1.2" + apache-md5: "npm:^1.0.6" + bcryptjs: "npm:^2.4.3" + uuid: "npm:^8.3.2" + checksum: 10c0/9bd935ec6819e05c5b689db2f97fd81a9a1a0957074b0cbecc94251a47dc0b654e89ff8da9d091bd9ef16351e13a1fb8dda0a2e9352fb31bf137f3dbf5b3b28b + languageName: node + linkType: hard + "http-cache-semantics@npm:3.8.1": version: 3.8.1 resolution: "http-cache-semantics@npm:3.8.1" @@ -26029,6 +26068,13 @@ __metadata: languageName: node linkType: hard +"unix-crypt-td-js@npm:^1.1.4": + version: 1.1.4 + resolution: "unix-crypt-td-js@npm:1.1.4" + checksum: 10c0/c4e3abd0d7ebcf39df7faff8be2cd137f477add743a2793c551682e04ec4e4f466e806a67e391d5a097229e4465b7cae4cb459990b9eb61dfe0b37d2388c6266 + languageName: node + linkType: hard + "unpipe@npm:1.0.0, unpipe@npm:~1.0.0": version: 1.0.0 resolution: "unpipe@npm:1.0.0" From 8d5f2ae22ac5a32d7987f6268ab6446e1ddeed0e Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Wed, 10 Jul 2024 15:29:40 +0200 Subject: [PATCH 09/25] fix(queue): fix http-auth type not found --- queue/src/basic-auth.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queue/src/basic-auth.ts b/queue/src/basic-auth.ts index 64f1c836d..6c176b1e9 100644 --- a/queue/src/basic-auth.ts +++ b/queue/src/basic-auth.ts @@ -1,7 +1,7 @@ import auth from 'http-auth'; import path from 'node:path'; -export const createBasicAuth = () => +export const createBasicAuth = (): ReturnType => auth.basic({ file: path.join(__dirname, '.htpasswd'), skipUser: true From 771b7ebd5d29ddfc1d6e99297e04f85e01907f8f Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Wed, 10 Jul 2024 15:53:50 +0200 Subject: [PATCH 10/25] feat(queue): remove .htpasswd and generate it later --- queue/src/.htpasswd | 1 - 1 file changed, 1 deletion(-) delete mode 100644 queue/src/.htpasswd diff --git a/queue/src/.htpasswd b/queue/src/.htpasswd deleted file mode 100644 index 0daeb4f9f..000000000 --- a/queue/src/.htpasswd +++ /dev/null @@ -1 +0,0 @@ -zlv@beta.gouv.fr:$apr1$6ts5byl3$575IQxAARDD6llSdjDpQM0 From e75d88445fe0f7d8f082ec52b428875031b5c160 Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Wed, 10 Jul 2024 20:52:32 +0200 Subject: [PATCH 11/25] test(e2e): upload cypress screenshots to github if they fail --- .github/workflows/e2e.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 0d89b50c6..8a1d3d280 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -39,3 +39,10 @@ jobs: CYPRESS_BASE_URL: ${{ vars.HOST }} CYPRESS_EMAIL: ${{ secrets.CYPRESS_EMAIL }} CYPRESS_PASSWORD: ${{ secrets.CYPRESS_PASSWORD }} + + - name: Upload screenshots + uses: actions/upload-artifact@4 + if: failure() + with: + name: cypress-screenshots + path: cypress/screenshots From f7a38dd6cb7d71c063de42ddb0e7a706baf59f08 Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Wed, 10 Jul 2024 21:07:05 +0200 Subject: [PATCH 12/25] ci(e2e): retain cypress screenshots for 7 days; fix action --- .github/workflows/e2e.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 8a1d3d280..50585eb0d 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -41,8 +41,9 @@ jobs: CYPRESS_PASSWORD: ${{ secrets.CYPRESS_PASSWORD }} - name: Upload screenshots - uses: actions/upload-artifact@4 + uses: actions/upload-artifact@v4 if: failure() with: name: cypress-screenshots path: cypress/screenshots + retention-days: 7 From 54ecfc2e17e0e2f4e7e80779ed74876301ec259f Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Wed, 10 Jul 2024 21:32:40 +0200 Subject: [PATCH 13/25] ci(e2e): fix screenshots path --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 50585eb0d..cbfb14310 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -45,5 +45,5 @@ jobs: if: failure() with: name: cypress-screenshots - path: cypress/screenshots + path: e2e/cypress/screenshots retention-days: 7 From d991606b27ba03994c014205afccf9285c9bf4ff Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Wed, 10 Jul 2024 22:20:33 +0200 Subject: [PATCH 14/25] test(e2e): wait for upload before setting the date --- e2e/cypress/e2e/campaign.cy.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e/cypress/e2e/campaign.cy.ts b/e2e/cypress/e2e/campaign.cy.ts index 9aa4dd8ef..e3e66552d 100644 --- a/e2e/cypress/e2e/campaign.cy.ts +++ b/e2e/cypress/e2e/campaign.cy.ts @@ -4,6 +4,7 @@ describe('Campaign', () => { cy.intercept('POST', Cypress.env('API') + '/housing/count').as( 'countHousings' ); + cy.intercept('POST', Cypress.env('API') + '/files').as('upload'); cy.logIn(); cy.wait(['@findHousings', '@countHousings']); @@ -26,6 +27,7 @@ describe('Campaign', () => { cy.get('input[type="file"]') .first() .selectFile('cypress/fixtures/logo.png'); + cy.wait('@upload'); cy.get('label') .contains(/^En date du/) From 0af1e74e34920c7a6a98625617bd3f82a742c005 Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Mon, 15 Jul 2024 09:42:39 +0200 Subject: [PATCH 15/25] ci(deploy): fix env var typo --- .github/workflows/review-app.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/review-app.yml b/.github/workflows/review-app.yml index c2126f2c6..f0accf4ae 100644 --- a/.github/workflows/review-app.yml +++ b/.github/workflows/review-app.yml @@ -63,9 +63,6 @@ jobs: clever env -a $APP_ALIAS | grep -v '^#' | tr -d '"' >> "$GITHUB_ENV" - name: Set environment variables - env: - E2E_EMAIL: ${{ secrets.E2E_EMAIL }} - E2E_PASSWORD: ${{ secrets.E2E_PASSWORD }} run: | clever env set AUTH_SECRET "secret" -a $APP_ALIAS clever env set CC_HEALTH_CHECK_PATH "/" -a $APP_ALIAS @@ -77,7 +74,7 @@ jobs: clever env set DATABASE_ENV "development" -a $APP_ALIAS clever env set DATABASE_URL $POSTGRESQL_ADDON_URI -a $APP_ALIAS clever env set E2E_EMAIL $E2E_EMAIL -a $APP_ALIAS - clever env set E2E_PASSWORD: $E2E_PASSWORD -a $APP_ALIAS + clever env set E2E_PASSWORD $E2E_PASSWORD -a $APP_ALIAS clever env set HOST "https://$APP_ALIAS.cleverapps.io" -a $APP_ALIAS clever env set METABASE_TOKEN "unused" -a $APP_ALIAS clever env set PORT "8080" -a $APP_ALIAS From 90347913f62614a172ec40ff0a339646af7df06a Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Thu, 11 Jul 2024 17:52:47 +0200 Subject: [PATCH 16/25] fix(server): housing creation should fetch data from df_housing_nat and df_owners_nat --- ...-housing-nat-2023.ts => df-housing-nat.ts} | 10 ++++---- ...df-owners-nat-2023.ts => df-owners-nat.ts} | 10 ++++---- ...-housing-nat-2023.ts => df-housing-nat.ts} | 10 ++++---- ...df-owners-nat-2023.ts => df-owners-nat.ts} | 10 ++++---- .../datafoncierHousingRepository.ts | 11 ++++----- .../datafoncierOwnersRepository.ts | 21 ++++++++--------- .../src/scripts/duplicate-database/script.sh | 1 - .../import-datafoncier-to-zlv/fix-geo.ts | 23 +++++++++---------- .../import-datafoncier-to-zlv/fix-plot_id.ts | 12 +++++----- .../import-datafoncier-to-zlv/index.ts | 18 +++++++-------- 10 files changed, 60 insertions(+), 66 deletions(-) rename server/src/infra/database/seeds/development/{df-housing-nat-2023.ts => df-housing-nat.ts} (95%) rename server/src/infra/database/seeds/development/{df-owners-nat-2023.ts => df-owners-nat.ts} (92%) rename server/src/infra/database/seeds/test/{df-housing-nat-2023.ts => df-housing-nat.ts} (95%) rename server/src/infra/database/seeds/test/{df-owners-nat-2023.ts => df-owners-nat.ts} (91%) diff --git a/server/src/infra/database/seeds/development/df-housing-nat-2023.ts b/server/src/infra/database/seeds/development/df-housing-nat.ts similarity index 95% rename from server/src/infra/database/seeds/development/df-housing-nat-2023.ts rename to server/src/infra/database/seeds/development/df-housing-nat.ts index 57f0cdf68..54edc91fd 100644 --- a/server/src/infra/database/seeds/development/df-housing-nat-2023.ts +++ b/server/src/infra/database/seeds/development/df-housing-nat.ts @@ -4,12 +4,12 @@ import { Knex } from 'knex'; import { genDatafoncierHousing } from '~/test/testFixtures'; import { Establishments } from '~/repositories/establishmentRepository'; -const DF_HOUSING_NAT_2023 = 'df_housing_nat_2023'; +const DF_HOUSING_NAT = 'df_housing_nat'; export async function seed(knex: Knex): Promise { - const exists = await knex.schema.hasTable(DF_HOUSING_NAT_2023); + const exists = await knex.schema.hasTable(DF_HOUSING_NAT); if (!exists) { - await knex.schema.createTable(DF_HOUSING_NAT_2023, (table) => { + await knex.schema.createTable(DF_HOUSING_NAT, (table) => { table.string('idlocal').notNullable(); table.string('idbat').notNullable(); table.string('idpar').notNullable(); @@ -140,7 +140,7 @@ export async function seed(knex: Knex): Promise { } // Deletes ALL existing entries - await knex(DF_HOUSING_NAT_2023).delete(); + await knex(DF_HOUSING_NAT).delete(); // Inserts seed entries const availableEstablishments = await Establishments(knex).where({ @@ -153,6 +153,6 @@ export async function seed(knex: Knex): Promise { const houses = Array.from({ length: 10 }, () => genDatafoncierHousing(geoCode) ); - await knex(DF_HOUSING_NAT_2023).insert(houses); + await knex(DF_HOUSING_NAT).insert(houses); }); } diff --git a/server/src/infra/database/seeds/development/df-owners-nat-2023.ts b/server/src/infra/database/seeds/development/df-owners-nat.ts similarity index 92% rename from server/src/infra/database/seeds/development/df-owners-nat-2023.ts rename to server/src/infra/database/seeds/development/df-owners-nat.ts index ceb25caa6..fa7e32827 100644 --- a/server/src/infra/database/seeds/development/df-owners-nat-2023.ts +++ b/server/src/infra/database/seeds/development/df-owners-nat.ts @@ -5,12 +5,12 @@ import { Knex } from 'knex'; import { genDatafoncierOwner } from '~/test/testFixtures'; import { DatafoncierHouses } from '~/repositories/datafoncierHousingRepository'; -const DF_OWNERS_NAT_2023 = 'df_owners_nat_2023'; +const DF_OWNERS_NAT = 'df_owners_nat'; export async function seed(knex: Knex): Promise { - const exists = await knex.schema.hasTable(DF_OWNERS_NAT_2023); + const exists = await knex.schema.hasTable(DF_OWNERS_NAT); if (!exists) { - await knex.schema.createTable(DF_OWNERS_NAT_2023, (table) => { + await knex.schema.createTable(DF_OWNERS_NAT, (table) => { table.string('idprodroit').notNullable(); table.string('idprocpte').notNullable(); table.string('idpersonne').notNullable(); @@ -78,7 +78,7 @@ export async function seed(knex: Knex): Promise { } // Deletes ALL existing entries - await knex(DF_OWNERS_NAT_2023).delete(); + await knex(DF_OWNERS_NAT).delete(); // Inserts seed entries const housings = await DatafoncierHouses(knex).select('idprocpte'); @@ -93,6 +93,6 @@ export async function seed(knex: Knex): Promise { } } ); - await knex(DF_OWNERS_NAT_2023).insert(owners); + await knex(DF_OWNERS_NAT).insert(owners); }); } diff --git a/server/src/infra/database/seeds/test/df-housing-nat-2023.ts b/server/src/infra/database/seeds/test/df-housing-nat.ts similarity index 95% rename from server/src/infra/database/seeds/test/df-housing-nat-2023.ts rename to server/src/infra/database/seeds/test/df-housing-nat.ts index f454e6f4a..d47861a52 100644 --- a/server/src/infra/database/seeds/test/df-housing-nat-2023.ts +++ b/server/src/infra/database/seeds/test/df-housing-nat.ts @@ -2,12 +2,12 @@ import { Knex } from 'knex'; import { genDatafoncierHousing } from '~/test/testFixtures'; -const DF_HOUSING_NAT_2023 = 'df_housing_nat_2023'; +const DF_HOUSING_NAT = 'df_housing_nat'; export async function seed(knex: Knex): Promise { - const exists = await knex.schema.hasTable(DF_HOUSING_NAT_2023); + const exists = await knex.schema.hasTable(DF_HOUSING_NAT); if (!exists) { - await knex.schema.createTable(DF_HOUSING_NAT_2023, (table) => { + await knex.schema.createTable(DF_HOUSING_NAT, (table) => { table.string('idlocal').notNullable(); table.string('idbat').notNullable(); table.string('idpar').notNullable(); @@ -138,9 +138,9 @@ export async function seed(knex: Knex): Promise { } // Deletes ALL existing entries - await knex(DF_HOUSING_NAT_2023).delete(); + await knex(DF_HOUSING_NAT).delete(); // Inserts seed entries const houses = Array.from({ length: 100 }, () => genDatafoncierHousing()); - await knex(DF_HOUSING_NAT_2023).insert(houses); + await knex(DF_HOUSING_NAT).insert(houses); } diff --git a/server/src/infra/database/seeds/test/df-owners-nat-2023.ts b/server/src/infra/database/seeds/test/df-owners-nat.ts similarity index 91% rename from server/src/infra/database/seeds/test/df-owners-nat-2023.ts rename to server/src/infra/database/seeds/test/df-owners-nat.ts index b185104d2..47207ca29 100644 --- a/server/src/infra/database/seeds/test/df-owners-nat-2023.ts +++ b/server/src/infra/database/seeds/test/df-owners-nat.ts @@ -2,12 +2,12 @@ import { Knex } from 'knex'; import { genDatafoncierOwner } from '~/test/testFixtures'; -const DF_OWNERS_NAT_2023 = 'df_owners_nat_2023'; +const DF_OWNERS_NAT = 'df_owners_nat'; export async function seed(knex: Knex): Promise { - const exists = await knex.schema.hasTable(DF_OWNERS_NAT_2023); + const exists = await knex.schema.hasTable(DF_OWNERS_NAT); if (!exists) { - await knex.schema.createTable(DF_OWNERS_NAT_2023, (table) => { + await knex.schema.createTable(DF_OWNERS_NAT, (table) => { table.string('idprodroit').notNullable(); table.string('idprocpte').notNullable(); table.string('idpersonne').notNullable(); @@ -75,9 +75,9 @@ export async function seed(knex: Knex): Promise { } // Deletes ALL existing entries - await knex(DF_OWNERS_NAT_2023).delete(); + await knex(DF_OWNERS_NAT).delete(); // Inserts seed entries const owners = Array.from({ length: 100 }, () => genDatafoncierOwner()); - await knex(DF_OWNERS_NAT_2023).insert(owners); + await knex(DF_OWNERS_NAT).insert(owners); } diff --git a/server/src/repositories/datafoncierHousingRepository.ts b/server/src/repositories/datafoncierHousingRepository.ts index 169e8d764..8c5145e82 100644 --- a/server/src/repositories/datafoncierHousingRepository.ts +++ b/server/src/repositories/datafoncierHousingRepository.ts @@ -1,20 +1,19 @@ import highland from 'highland'; -import Stream = Highland.Stream; import { logger } from '~/infra/logger'; import db from '~/infra/database'; import { DatafoncierHousing } from '@zerologementvacant/shared'; -import { getYear } from 'date-fns'; +import Stream = Highland.Stream; const FIELDS = ['*']; -export const datafoncierHousingTable = `df_housing_nat_${getYear(new Date()) - 1}`; +export const datafoncierHousingTable = 'df_housing_nat'; export const DatafoncierHouses = (transaction = db) => transaction(datafoncierHousingTable); class DatafoncierHousingRepository { async find( - where: Partial, + where: Partial ): Promise { const housingList = await DatafoncierHouses() .where(where) @@ -23,7 +22,7 @@ class DatafoncierHousingRepository { } async findOne( - where: Partial, + where: Partial ): Promise { const housing = await DatafoncierHouses() .where(where) @@ -38,7 +37,7 @@ class DatafoncierHousingRepository { const query = DatafoncierHouses() .select(FIELDS) .where({ - ccthp: 'L', + ccthp: 'L' }) .whereIn('dteloctxt', ['APPARTEMENT', 'MAISON']) .stream(); diff --git a/server/src/repositories/datafoncierOwnersRepository.ts b/server/src/repositories/datafoncierOwnersRepository.ts index 2fa082227..aac04ce3c 100644 --- a/server/src/repositories/datafoncierOwnersRepository.ts +++ b/server/src/repositories/datafoncierOwnersRepository.ts @@ -4,7 +4,7 @@ import { DatafoncierOwner, DatafoncierOwnerSortApi, ownerDatafoncierSchema, - validator, + validator } from '~/scripts/shared'; import { Knex } from 'knex'; import { ownerMatchTable } from './ownerMatchRepository'; @@ -12,7 +12,6 @@ import { OwnerDBO, ownerTable, parseOwnerApi } from './ownerRepository'; import { OwnerApi } from '~/models/OwnerApi'; import fp from 'lodash/fp'; import { sortQuery } from '~/models/SortApi'; -import { getYear } from 'date-fns'; const FIELDS = [ 'idprodroit', @@ -24,10 +23,10 @@ const FIELDS = [ 'ddenom', 'jdatnss', 'catpro2txt', - 'catpro3txt', + 'catpro3txt' ]; -export const datafoncierOwnersTable = `df_owners_nat_${getYear(new Date()) - 1}`; +export const datafoncierOwnersTable = 'df_owners_nat'; export const DatafoncierOwners = (transaction = db) => transaction(datafoncierOwnersTable); @@ -44,7 +43,7 @@ class DatafoncierOwnersRepository { const subquery = DatafoncierOwners() .distinctOn('idpersonne') .where((whereBuilder) => - whereBuilder.whereNull('ccogrm').orWhereIn('ccogrm', ['0', '7', '8']), + whereBuilder.whereNull('ccogrm').orWhereIn('ccogrm', ['0', '7', '8']) ) .select('idpersonne'); // Sélectionnez uniquement 'idpersonne' pour le décompte distinct @@ -63,7 +62,7 @@ class DatafoncierOwnersRepository { .join( ownerMatchTable, `${ownerMatchTable}.idpersonne`, - `${datafoncierOwnersTable}.idpersonne`, + `${datafoncierOwnersTable}.idpersonne` ) .join(ownerTable, `${ownerTable}.id`, `${ownerMatchTable}.owner_id`) .orderBy(`${datafoncierOwnersTable}.dnulp`); @@ -84,7 +83,7 @@ class DatafoncierOwnersRepository { .select(FIELDS) .distinctOn('idpersonne') .where((whereBuilder) => - whereBuilder.whereNull('ccogrm').orWhereIn('ccogrm', ['0', '7', '8']), + whereBuilder.whereNull('ccogrm').orWhereIn('ccogrm', ['0', '7', '8']) ) // Avoid importing owners that have no address at all .modify(hasAddress()) @@ -93,15 +92,15 @@ class DatafoncierOwnersRepository { sortQuery(opts?.sort, { keys: { idprocpte: (query) => - query.orderBy('idprocpte', opts?.sort?.idprocpte), + query.orderBy('idprocpte', opts?.sort?.idprocpte) }, - default: (query) => query.orderBy('idpersonne'), - }), + default: (query) => query.orderBy('idpersonne') + }) ) .stream(); return highland(query).map( - validator.validate(ownerDatafoncierSchema), + validator.validate(ownerDatafoncierSchema) ); } } diff --git a/server/src/scripts/duplicate-database/script.sh b/server/src/scripts/duplicate-database/script.sh index b5d1988f0..32d257b9c 100755 --- a/server/src/scripts/duplicate-database/script.sh +++ b/server/src/scripts/duplicate-database/script.sh @@ -42,7 +42,6 @@ wait # Exclude datafoncier pg_restore --list ${backup_file_name} > toc.list grep -Ev "df_housing_nat|df_owners_nat" toc.list > filtered_toc.list -grep -Ev "df_housing_nat_[0-9]{4}|df_owners_[0-9]{4}" filtered_toc.list > final_toc.list wait diff --git a/server/src/scripts/import-datafoncier-to-zlv/fix-geo.ts b/server/src/scripts/import-datafoncier-to-zlv/fix-geo.ts index 9308c6b48..88a19c6d7 100644 --- a/server/src/scripts/import-datafoncier-to-zlv/fix-geo.ts +++ b/server/src/scripts/import-datafoncier-to-zlv/fix-geo.ts @@ -1,4 +1,4 @@ -import { SingleBar, Presets } from 'cli-progress'; +import { Presets, SingleBar } from 'cli-progress'; import { parse } from 'csv-parse'; import { Map } from 'immutable'; import { execSync } from 'node:child_process'; @@ -10,11 +10,10 @@ import { logger } from '~/infra/logger'; const progressBarBAN = new SingleBar({}, Presets.shades_classic); const progressBarUpdate = new SingleBar({}, Presets.shades_classic); -const YEAR = '2023'; const BAN_FILE_LOCATION = '/Volumes/Zéro_Logement_Vacant/BAN/adresses-france-BAL.csv'; const result = execSync(`wc -l < "${BAN_FILE_LOCATION}"`, { - encoding: 'utf-8', + encoding: 'utf-8' }); const TOTAL_BAN_DATA_LINES = parseInt(result.trim(), 10); @@ -29,7 +28,7 @@ const processRow = async (housing: any) => { const banItem = banData.get(ban_id); if (banItem?.long && banItem?.lat) { await db.raw( - `update fast_housing set longitude='${banItem?.long}', latitude='${banItem?.lat}' where local_id='${housing.local_id}'`, + `update fast_housing set longitude='${banItem?.long}', latitude='${banItem?.lat}' where local_id='${housing.local_id}'` ); } } @@ -49,14 +48,14 @@ function transform(transformFunction: { transform: function ( row: any, encoding: any, - callback: (arg0: null, arg1: any) => any, + callback: (arg0: null, arg1: any) => any ) { if (isFirstCol) { isFirstCol = false; return callback(null, row); } transformFunction(row, callback); - }, + } }); } @@ -70,12 +69,12 @@ const main = async () => { .pipe( transform(function ( row: { [s: string]: unknown } | ArrayLike, - callback: (arg0: null, arg1: unknown[]) => void, + callback: (arg0: null, arg1: unknown[]) => void ) { // Ignorer le premier élément de chaque ligne const values = Object.values(row); callback(null, values.slice(1)); - }), + }) ) .on('data', (data: string[]) => { progressBarBAN.increment(); @@ -86,14 +85,14 @@ const main = async () => { banData = banData.set(uid_adresse, { long: parseFloat(long), - lat: parseFloat(lat), + lat: parseFloat(lat) }); }) .on('end', async () => { logger.info('Update data...'); const count = await db.raw( - `select count(*) from fast_housing where latitude is null`, + `select count(*) from fast_housing where latitude is null` ); progressBarUpdate.start(parseInt(count.rows[0].count), 0); @@ -102,8 +101,8 @@ const main = async () => { .raw( `SELECT local_id, ban_id, idpar FROM fast_housing fh - JOIN df_housing_nat_${YEAR} df ON fh.local_id = df.idLocal - WHERE fh.latitude IS NULL`, + JOIN df_housing_nat df ON fh.local_id = df.idLocal + WHERE fh.latitude IS NULL` ) .stream(); diff --git a/server/src/scripts/import-datafoncier-to-zlv/fix-plot_id.ts b/server/src/scripts/import-datafoncier-to-zlv/fix-plot_id.ts index d3f556590..269bf33bb 100644 --- a/server/src/scripts/import-datafoncier-to-zlv/fix-plot_id.ts +++ b/server/src/scripts/import-datafoncier-to-zlv/fix-plot_id.ts @@ -1,6 +1,6 @@ import { logger } from '~/infra/logger'; import db from '~/infra/database/'; -import { SingleBar, Presets } from 'cli-progress'; +import { Presets, SingleBar } from 'cli-progress'; const progressBar = new SingleBar({}, Presets.shades_classic); @@ -9,7 +9,7 @@ const processRow = async (housing: any) => { progressBar.increment(); const doProcess = async () => { await db.raw( - `UPDATE fast_housing SET plot_id='${housing.idpar}' WHERE local_id='${housing.local_id}'`, + `UPDATE fast_housing SET plot_id='${housing.idpar}' WHERE local_id='${housing.local_id}'` ); resolve(); }; @@ -19,11 +19,11 @@ const processRow = async (housing: any) => { const main = async () => { logger.info( - 'Importing datafoncier raw data to ZLV tables (missing plot_id)...', + 'Importing datafoncier raw data to ZLV tables (missing plot_id)...' ); const count = await db.raw( - `select count(*) from fast_housing where source = 'datafoncier-import' and plot_id IS NULL`, + `select count(*) from fast_housing where source = 'datafoncier-import' and plot_id IS NULL` ); progressBar.start(parseInt(count.rows[0].count), 0); @@ -31,8 +31,8 @@ const main = async () => { .raw( `SELECT local_id, idpar FROM fast_housing fh - JOIN df_housing_nat_2023 df ON fh.local_id = df.idLocal - WHERE source = 'datafoncier-import' and plot_id IS NULL`, + JOIN df_housing_nat df ON fh.local_id = df.idLocal + WHERE source = 'datafoncier-import' and plot_id IS NULL` ) .stream(); diff --git a/server/src/scripts/import-datafoncier-to-zlv/index.ts b/server/src/scripts/import-datafoncier-to-zlv/index.ts index dd2ebcfbc..f9dc4aa69 100644 --- a/server/src/scripts/import-datafoncier-to-zlv/index.ts +++ b/server/src/scripts/import-datafoncier-to-zlv/index.ts @@ -2,7 +2,7 @@ import { logger } from '~/infra/logger'; import { toHousingRecordApi, toOwnerApi } from '../shared'; import db from '~/infra/database/'; -import { SingleBar, Presets } from 'cli-progress'; +import { Presets, SingleBar } from 'cli-progress'; import ownerMatchRepository from '~/repositories/ownerMatchRepository'; import ownerRepository from '~/repositories/ownerRepository'; import { HousingOwners } from '~/repositories/housingOwnerRepository'; @@ -10,18 +10,16 @@ import housingRepository from '~/repositories/housingRepository'; const progressBar = new SingleBar({}, Presets.shades_classic); -const YEAR = '2023'; - const processRow = async (dfHousing: any) => { progressBar.increment(); const doProcess = async () => { const housing = toHousingRecordApi( { source: 'datafoncier-import' }, - dfHousing, + dfHousing ); const dfOwner = await db.raw( - `SELECT * FROM df_owners_nat_${YEAR} WHERE idlocal='${housing.localId}'`, + `SELECT * FROM df_owners_nat WHERE idlocal='${housing.localId}'` ); if (dfOwner.rows.length === 0) { @@ -29,7 +27,7 @@ const processRow = async (dfHousing: any) => { } const ownerMatch = await ownerMatchRepository.findOne({ - idpersonne: dfOwner.rows[0].idpersonne, + idpersonne: dfOwner.rows[0].idpersonne }); let owner; @@ -53,7 +51,7 @@ const processRow = async (dfHousing: any) => { owner_id: owner.id, housing_id: housing.id, housing_geo_code: housing.geoCode, - rank: 1, + rank: 1 }); } catch (e: any) { return; @@ -67,7 +65,7 @@ const main = async () => { logger.info('Importing datafoncier raw data to ZLV tables...'); const count = await db.raw(`SELECT count(df.*) - FROM df_housing_nat_${YEAR} df + FROM df_housing_nat df WHERE NOT EXISTS ( SELECT 1 FROM fast_housing fh @@ -80,13 +78,13 @@ const main = async () => { .raw( ` SELECT df.* - FROM df_housing_nat_${YEAR} df + FROM df_housing_nat df WHERE NOT EXISTS ( SELECT 1 FROM fast_housing fh WHERE df.idlocal = fh.local_id ) - `, + ` ) .stream(); From 7da304ec6938639c59a11936fbb60bf214d0f3c4 Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Mon, 15 Jul 2024 09:33:02 +0200 Subject: [PATCH 17/25] build: fix .talismanrc --- .talismanrc | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.talismanrc b/.talismanrc index 1f9ef3491..93ba18757 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,9 +1,6 @@ fileignoreconfig: - filename: .github/workflows/review-app.yml checksum: 78d9abc578bda1e6622a67acfd199ffba381ad98cfeeb5b31bb1fe74c95fc3c1 -version: "" -hub/workflows/e2e.yml - checksum: 1994a562d1d57521c10078640cd49fdccf7b3baecbaf46f3ed383ada996b5c90 - filename: .github/workflows/github-actions.yml checksum: b1a53b557b6b2ac4c57f6afce9e902582f5589f97762a963da587496a455c0c1 - filename: .github/workflows/review-app.yml @@ -135,11 +132,3 @@ allowed_patterns: - key=\{.+\} - keyof version: "1.0" -red/src/utils/s3.ts - checksum: 96fa9978f136699921498fe90d02c495d4c1d27f47c96e64b646b219f29204c1 -scopeconfig: -- scope: node -allowed_patterns: -- key=\{.+\} -- keyof -version: "1.0" From 5ac317ddf69224538463cc7db8d9c0386b7337c8 Mon Sep 17 00:00:00 2001 From: JuGuuu <101636995+JuGuuu@users.noreply.github.com> Date: Tue, 16 Jul 2024 15:02:48 +0200 Subject: [PATCH 18/25] Modif_wording_resources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Textes de chaque rubrique raccourcis pour améliorer l'accessibilité --- .../src/views/Resources/ResourcesView.tsx | 42 ++++--------------- 1 file changed, 7 insertions(+), 35 deletions(-) diff --git a/frontend/src/views/Resources/ResourcesView.tsx b/frontend/src/views/Resources/ResourcesView.tsx index 84bb1433a..78bc3d133 100644 --- a/frontend/src/views/Resources/ResourcesView.tsx +++ b/frontend/src/views/Resources/ResourcesView.tsx @@ -51,12 +51,6 @@ const ResourcesView = () => { return ( - - Parcourez les différentes rubriques pour trouver les informations et - documents utiles dans votre stratégie de lutte contre la vacance. - Retrouvez également en bas de cette page les trois étapes clés pour - prendre en main ZLV ! - { icon="fr-icon-folder-2-fill" iconStyle={styles.iconResource} > - Zéro Logement Vacant vous propose une sélection de ressources pour - comprendre les données LOVAC, connaître le profil des - propriétaires et échanger avec eux, mais aussi s’informer sur les - dispositifs d’aides ou proposer des missions de service civique. + Découvrez une sélection de ressources autour de la lutte contre la vacance. @@ -78,10 +69,7 @@ const ResourcesView = () => { icon="fr-icon-group-fill" iconStyle={styles.iconCommunity} > - Utiliser Zéro Logement Vacant, c’est aussi faire partie d’une - communauté de collectivités utilisatrices de la solution... Pour - échanger sur la plateforme Rencontre des Territoires et participer - au club des collectivités utilisatrices de ZLV, c’est par ici ! + La plateforme d’échanges Rencontre des Territoires et les Clubs ZLV, c’est par ici ! @@ -91,10 +79,7 @@ const ResourcesView = () => { icon="fr-icon-question-mark" iconStyle={styles.iconHelp} > - Vous avez une question sur la solution ZLV ou sur les données - utilisées ? Vous ne savez pas comment créer une campagne ou mettre - à jour des dossiers ? Vous trouverez dans ce centre d’aide toutes - les réponses à vos questions les plus fréquentes ! + Une question sur ZLV ? La réponse est sûrement dans le centre d’aide. @@ -104,10 +89,7 @@ const ResourcesView = () => { icon="fr-icon-calendar-fill" iconStyle={styles.iconAgenda} > - Vous souhaitez être accompagné dans la création d’une campagne ou - la mise à jour des dossiers ? Vous souhaitez échanger avec nous - autour de votre stratégie de lutte contre la vacance ou nous faire - un retour d’expérience ? Prenez rendez-vous avec nous ! + Besoin d’échanger avec nous ? Prenez rendez-vous en visio ! @@ -123,10 +105,7 @@ const ResourcesView = () => { linkHref="https://zlv.notion.site/Int-grer-un-p-rim-tre-5c7cf0d51f20448bb1316405adbb4a37" icon="fr-icon-road-map-fill" > - Vous souhaitez cibler les logements sur un périmètre en - particulier, comme un dispositif opérationnel ou un quartier ? - Nous vous expliquons ici comment intégrer vos périmètres dans la - solution ! + Découvrez comment cibler les logements au sein d’un périmètre géographique. @@ -135,11 +114,7 @@ const ResourcesView = () => { linkHref="https://zlv.notion.site/R-diger-un-courrier-15e88e19d2bc404eaf371ddcb4ca42c5" icon="fr-icon-mail-fill" > - Vous voulez rédiger un courrier mais ne savez pas par où commencer - ? Vous souhaitez mobiliser les propriétaires et cherchez les - arguments à mettre en avant dans le courrier ? Vous avez besoin - d’un modèle adapté à un contexte particulier ? Laissez-vous guider - ! + Accédez à des conseils pour écrire vos courriers et à des modèles déjà rédigés. @@ -149,10 +124,7 @@ const ResourcesView = () => { linkHrefTarget="_self" icon="fr-icon-git-merge-line" > - Statut “Premier contact” ou “Suivi en cours” ? Il peut être - compliqué de s’y retrouver dans les statuts des dossiers sur ZLV… - Découvrez ici l’ensemble des statuts et sous-statuts que vous - pouvez appliquer aux dossiers. + Consultez l’ensemble des statuts que vous pouvez appliquer aux logements. From acb410e9d3560e6943fff89c6f3f5cff66cebf2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Guillois?= Date: Tue, 16 Jul 2024 16:54:39 +0200 Subject: [PATCH 19/25] feat: add pagination to campaign recipients list --- .../Campaign/CampaignRecipients.tsx | 89 +++++++++++++++++-- 1 file changed, 80 insertions(+), 9 deletions(-) diff --git a/frontend/src/components/Campaign/CampaignRecipients.tsx b/frontend/src/components/Campaign/CampaignRecipients.tsx index 132aab15f..1d3a0e61f 100644 --- a/frontend/src/components/Campaign/CampaignRecipients.tsx +++ b/frontend/src/components/Campaign/CampaignRecipients.tsx @@ -1,8 +1,9 @@ import Badge from '@codegouvfr/react-dsfr/Badge'; import Table from '@codegouvfr/react-dsfr/Table'; +import { Pagination as DSFRPagination } from '../_dsfr'; import Typography from '@mui/material/Typography'; import Grid from '@mui/material/Unstable_Grid2'; -import { ReactNode } from 'react'; +import { ReactNode, useState } from 'react'; import { Campaign } from '../../models/Campaign'; import { useHousingList } from '../../hooks/useHousingList'; @@ -12,25 +13,57 @@ import AppLink from '../_app/AppLink/AppLink'; import { Housing } from '../../models/Housing'; import { useRemoveCampaignHousingMutation } from '../../services/campaign.service'; import ConfirmationModal from '../modals/ConfirmationModal/ConfirmationModal'; +import { Pagination } from '@zerologementvacant/models'; +import { DefaultPagination } from '../../store/reducers/housingReducer'; +import { usePagination } from '../../hooks/usePagination'; +import Button from '@codegouvfr/react-dsfr/Button'; +import { useCountHousingQuery } from '../../services/housing.service'; interface Props { campaign: Campaign; } function CampaignRecipients(props: Props) { + const [pagination, setPagination] = useState(DefaultPagination); + const filters = { + campaignIds: [props.campaign.id] + }; const { housingList } = useHousingList({ - filters: { - campaignIds: [props.campaign.id], - }, + filters, + pagination }); + const { data: count } = useCountHousingQuery(filters); + const filteredCount = count?.housing ?? 0; + + const { pageCount, hasPagination } = usePagination({ + ...pagination, + count: filteredCount + }); + + const changePerPage = (perPage: number) => { + setPagination({ + ...pagination, + page: 1, + perPage + }); + }; + + const changePage = (page: number) => { + console.log(page); + setPagination({ + ...pagination, + page + }); + }; + const [removeCampaignHousing] = useRemoveCampaignHousingMutation(); function removeHousing(housing: Housing): void { removeCampaignHousing({ campaignId: props.campaign.id, all: false, ids: [housing.id], - filters: {}, + filters: {} }); } @@ -46,10 +79,10 @@ function CampaignRecipients(props: Props) { 'Propriétaire principal', 'Adresse BAN du propriétaire', 'Complément d’adresse', - null, + null ]; const data: ReactNode[][] = (housingList ?? []).map((housing, i) => [ - `# ${i}`, + `# ${i + 1 + (pagination.page - 1) * pagination.perPage}`, removeHousing(housing)} @@ -97,12 +130,50 @@ function CampaignRecipients(props: Props) { Vous êtes sur le point de supprimer ce destinataire de la campagne. - , + ]); return ( + {hasPagination && ( + <> +
+ +
+
+ + + +
+ + )} ); } From b992f5ed4a79032e3847e3375d9099b6f813d67a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Guillois?= Date: Tue, 16 Jul 2024 16:54:39 +0200 Subject: [PATCH 20/25] feat: add pagination to campaign recipients list --- .../Campaign/CampaignRecipients.tsx | 87 +++++++++++++++++-- 1 file changed, 79 insertions(+), 8 deletions(-) diff --git a/frontend/src/components/Campaign/CampaignRecipients.tsx b/frontend/src/components/Campaign/CampaignRecipients.tsx index 132aab15f..80f8da43b 100644 --- a/frontend/src/components/Campaign/CampaignRecipients.tsx +++ b/frontend/src/components/Campaign/CampaignRecipients.tsx @@ -1,8 +1,9 @@ import Badge from '@codegouvfr/react-dsfr/Badge'; import Table from '@codegouvfr/react-dsfr/Table'; +import { Pagination as DSFRPagination } from '../_dsfr'; import Typography from '@mui/material/Typography'; import Grid from '@mui/material/Unstable_Grid2'; -import { ReactNode } from 'react'; +import { ReactNode, useState } from 'react'; import { Campaign } from '../../models/Campaign'; import { useHousingList } from '../../hooks/useHousingList'; @@ -12,18 +13,50 @@ import AppLink from '../_app/AppLink/AppLink'; import { Housing } from '../../models/Housing'; import { useRemoveCampaignHousingMutation } from '../../services/campaign.service'; import ConfirmationModal from '../modals/ConfirmationModal/ConfirmationModal'; +import { Pagination } from '@zerologementvacant/models'; +import { DefaultPagination } from '../../store/reducers/housingReducer'; +import { usePagination } from '../../hooks/usePagination'; +import Button from '@codegouvfr/react-dsfr/Button'; +import { useCountHousingQuery } from '../../services/housing.service'; interface Props { campaign: Campaign; } function CampaignRecipients(props: Props) { + const [pagination, setPagination] = useState(DefaultPagination); + const filters = { + campaignIds: [props.campaign.id] + }; const { housingList } = useHousingList({ - filters: { - campaignIds: [props.campaign.id], - }, + filters, + pagination }); + const { data: count } = useCountHousingQuery(filters); + const filteredCount = count?.housing ?? 0; + + const { pageCount, hasPagination } = usePagination({ + ...pagination, + count: filteredCount + }); + + const changePerPage = (perPage: number) => { + setPagination({ + ...pagination, + page: 1, + perPage + }); + }; + + const changePage = (page: number) => { + console.log(page); + setPagination({ + ...pagination, + page + }); + }; + const [removeCampaignHousing] = useRemoveCampaignHousingMutation(); function removeHousing(housing: Housing): void { removeCampaignHousing({ @@ -46,10 +79,10 @@ function CampaignRecipients(props: Props) { 'Propriétaire principal', 'Adresse BAN du propriétaire', 'Complément d’adresse', - null, + null ]; const data: ReactNode[][] = (housingList ?? []).map((housing, i) => [ - `# ${i}`, + `# ${i + 1 + (pagination.page - 1) * pagination.perPage}`, removeHousing(housing)} @@ -97,12 +130,50 @@ function CampaignRecipients(props: Props) { Vous êtes sur le point de supprimer ce destinataire de la campagne. - , + ]); return (
+ {hasPagination && ( + <> +
+ +
+
+ + + +
+ + )} ); } From 6c942c846ee7a355df929aa0b41f5ac6c6cce980 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Guillois?= Date: Tue, 16 Jul 2024 17:02:21 +0200 Subject: [PATCH 21/25] chore: change prettier default rule --- .prettierrc.json | 2 +- .../components/Campaign/CampaignRecipients.tsx | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.prettierrc.json b/.prettierrc.json index ef129409e..2fd49b29a 100644 --- a/.prettierrc.json +++ b/.prettierrc.json @@ -1,5 +1,5 @@ { - "trailingComma": "none", + "trailingComma": "all", "printWidth": 80, "singleQuote": true } diff --git a/frontend/src/components/Campaign/CampaignRecipients.tsx b/frontend/src/components/Campaign/CampaignRecipients.tsx index 80f8da43b..b03b7e122 100644 --- a/frontend/src/components/Campaign/CampaignRecipients.tsx +++ b/frontend/src/components/Campaign/CampaignRecipients.tsx @@ -26,11 +26,11 @@ interface Props { function CampaignRecipients(props: Props) { const [pagination, setPagination] = useState(DefaultPagination); const filters = { - campaignIds: [props.campaign.id] + campaignIds: [props.campaign.id], }; const { housingList } = useHousingList({ filters, - pagination + pagination, }); const { data: count } = useCountHousingQuery(filters); @@ -38,14 +38,14 @@ function CampaignRecipients(props: Props) { const { pageCount, hasPagination } = usePagination({ ...pagination, - count: filteredCount + count: filteredCount, }); const changePerPage = (perPage: number) => { setPagination({ ...pagination, page: 1, - perPage + perPage, }); }; @@ -53,7 +53,7 @@ function CampaignRecipients(props: Props) { console.log(page); setPagination({ ...pagination, - page + page, }); }; @@ -79,7 +79,7 @@ function CampaignRecipients(props: Props) { 'Propriétaire principal', 'Adresse BAN du propriétaire', 'Complément d’adresse', - null + null, ]; const data: ReactNode[][] = (housingList ?? []).map((housing, i) => [ `# ${i + 1 + (pagination.page - 1) * pagination.perPage}`, @@ -121,7 +121,7 @@ function CampaignRecipients(props: Props) { iconId: 'fr-icon-close-line', priority: 'tertiary', size: 'small', - title: 'Supprimer le propriétaire' + title: 'Supprimer le propriétaire', }} title="Suppression d’un propriétaire" onSubmit={() => removeHousing(housing)} @@ -130,7 +130,7 @@ function CampaignRecipients(props: Props) { Vous êtes sur le point de supprimer ce destinataire de la campagne. - + , ]); return ( From be42e99c52a17f30aed730b06a7694e03c632d9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20GUILLOIS?= Date: Tue, 16 Jul 2024 17:31:03 +0200 Subject: [PATCH 22/25] Update ResourcesView.tsx Remove unused import --- frontend/src/views/Resources/ResourcesView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/views/Resources/ResourcesView.tsx b/frontend/src/views/Resources/ResourcesView.tsx index 78bc3d133..73fdab5b1 100644 --- a/frontend/src/views/Resources/ResourcesView.tsx +++ b/frontend/src/views/Resources/ResourcesView.tsx @@ -3,7 +3,7 @@ import styles from './resources.module.scss'; import MainContainer from '../../components/MainContainer/MainContainer'; import Tile from '@codegouvfr/react-dsfr/Tile'; import classNames from 'classnames'; -import { Col, Container, Icon, Row, Text } from '../../components/_dsfr'; +import { Col, Container, Icon, Row } from '../../components/_dsfr'; import Typography from '@mui/material/Typography'; interface Props { From b1c8ce984b00f8bf05d2cecd5b0106d49aa662b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Guillois?= Date: Thu, 18 Jul 2024 11:40:15 +0200 Subject: [PATCH 23/25] feat: create visitor account on staging env --- .talismanrc | 11 ++++++++--- .../seeds/development/20240404235457_users.ts | 9 +++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.talismanrc b/.talismanrc index 93ba18757..030292893 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,6 +1,4 @@ fileignoreconfig: -- filename: .github/workflows/review-app.yml - checksum: 78d9abc578bda1e6622a67acfd199ffba381ad98cfeeb5b31bb1fe74c95fc3c1 - filename: .github/workflows/github-actions.yml checksum: b1a53b557b6b2ac4c57f6afce9e902582f5589f97762a963da587496a455c0c1 - filename: .github/workflows/review-app.yml @@ -113,7 +111,7 @@ fileignoreconfig: - filename: server/src/infra/database/scripts/002-load-establishments_direction_territoriale.sql checksum: b2db25db2503ffb08d08e232fa5aa319caa0bacca03b5af067038154bb5393ca - filename: server/src/infra/database/seeds/development/20240404235457_users.ts - checksum: f3fabfbe62c13a614236fd7c9474c78de726c44515e8f116f3532765907c218c + checksum: 85407878a244313b70b8a60b5c7d3e41c33df83629866437cbca3a309ab6e1bc - filename: server/src/infra/database/seeds/development/20240627141242_e2e.ts checksum: 56a46923a8cffefb7e70949b80718dc56ae2f3dd28f5057cb102461fe2fa10eb - filename: server/src/routers/unprotected.ts @@ -132,3 +130,10 @@ allowed_patterns: - key=\{.+\} - keyof version: "1.0" +85ac09fdcf3978c64ff996c2d0e8731431ecb2c0 +scopeconfig: +- scope: node +allowed_patterns: +- key=\{.+\} +- keyof +version: "1.0" diff --git a/server/src/infra/database/seeds/development/20240404235457_users.ts b/server/src/infra/database/seeds/development/20240404235457_users.ts index 8e65eb0e8..18386a18d 100644 --- a/server/src/infra/database/seeds/development/20240404235457_users.ts +++ b/server/src/infra/database/seeds/development/20240404235457_users.ts @@ -43,6 +43,15 @@ export async function seed(knex: Knex): Promise { activatedAt: new Date(), role: UserRoles.Admin, }, + { + id: uuidv4(), + email: 'test.visitor@zlv.fr', + password: bcrypt.hashSync('test'), + firstName: 'Test', + lastName: 'Visitor', + activatedAt: new Date(), + role: UserRoles.Visitor, + }, { id: uuidv4(), email: 'admin@zerologementvacant.beta.gouv.fr', From 9a73662bb4445ac19edd58b0c96ed2817b736475 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Guillois?= Date: Thu, 18 Jul 2024 16:37:46 +0200 Subject: [PATCH 24/25] refactoring: move changePage and changePerPage to usePagination --- .prettierrc.json | 2 +- .../Campaign/CampaignRecipients.tsx | 21 +++--------------- .../components/HousingList/HousingList.tsx | 20 +++-------------- .../HousingAdditionalOwnerSearchResults.tsx | 15 ++++++++----- frontend/src/hooks/usePagination.tsx | 22 +++++++++++++++++-- 5 files changed, 36 insertions(+), 44 deletions(-) diff --git a/.prettierrc.json b/.prettierrc.json index 2fd49b29a..ef129409e 100644 --- a/.prettierrc.json +++ b/.prettierrc.json @@ -1,5 +1,5 @@ { - "trailingComma": "all", + "trailingComma": "none", "printWidth": 80, "singleQuote": true } diff --git a/frontend/src/components/Campaign/CampaignRecipients.tsx b/frontend/src/components/Campaign/CampaignRecipients.tsx index b03b7e122..8506309b1 100644 --- a/frontend/src/components/Campaign/CampaignRecipients.tsx +++ b/frontend/src/components/Campaign/CampaignRecipients.tsx @@ -36,27 +36,12 @@ function CampaignRecipients(props: Props) { const { data: count } = useCountHousingQuery(filters); const filteredCount = count?.housing ?? 0; - const { pageCount, hasPagination } = usePagination({ - ...pagination, + const { pageCount, hasPagination, changePerPage, changePage} = usePagination({ + pagination, + setPagination, count: filteredCount, }); - const changePerPage = (perPage: number) => { - setPagination({ - ...pagination, - page: 1, - perPage, - }); - }; - - const changePage = (page: number) => { - console.log(page); - setPagination({ - ...pagination, - page, - }); - }; - const [removeCampaignHousing] = useRemoveCampaignHousingMutation(); function removeHousing(housing: Housing): void { removeCampaignHousing({ diff --git a/frontend/src/components/HousingList/HousingList.tsx b/frontend/src/components/HousingList/HousingList.tsx index 20fb490ac..1a8545714 100644 --- a/frontend/src/components/HousingList/HousingList.tsx +++ b/frontend/src/components/HousingList/HousingList.tsx @@ -84,26 +84,12 @@ const HousingList = ({ const { data: count } = useCountHousingQuery(filters); const filteredCount = count?.housing ?? 0; - const { pageCount, rowNumber, hasPagination } = usePagination({ - ...pagination, + const { pageCount, hasPagination, rowNumber, changePerPage, changePage} = usePagination({ + pagination, + setPagination, count: filteredCount, }); - const changePerPage = (perPage: number) => { - setPagination({ - ...pagination, - page: 1, - perPage, - }); - }; - - const changePage = (page: number) => { - setPagination({ - ...pagination, - page, - }); - }; - const onSort = (sort: HousingSort) => { setSort(sort); setPagination({ diff --git a/frontend/src/components/modals/HousingOwnersModal/HousingAdditionalOwnerSearchResults.tsx b/frontend/src/components/modals/HousingOwnersModal/HousingAdditionalOwnerSearchResults.tsx index 2c5a181a7..4314c33ac 100644 --- a/frontend/src/components/modals/HousingOwnersModal/HousingAdditionalOwnerSearchResults.tsx +++ b/frontend/src/components/modals/HousingOwnersModal/HousingAdditionalOwnerSearchResults.tsx @@ -1,11 +1,13 @@ -import { Col, Pagination, Row, Table, Text } from '../../_dsfr'; +import { Col, Pagination as DSFRPagination, Row, Table, Text } from '../../_dsfr'; import { format } from 'date-fns'; import { displayCount } from '../../../utils/stringUtils'; import { Owner } from '../../../models/Owner'; import { usePagination } from '../../../hooks/usePagination'; import { useFindOwnersQuery } from '../../../services/owner.service'; import { useAppDispatch, useAppSelector } from '../../../hooks/useStore'; -import housingSlice from '../../../store/reducers/housingReducer'; +import housingSlice, { DefaultPagination } from '../../../store/reducers/housingReducer'; +import { useState } from 'react'; +import { Pagination } from '@zerologementvacant/models'; interface Props { onSelect: (owner: Owner) => void; @@ -22,11 +24,12 @@ const HousingAdditionalOwnerSearchResults = ({ onSelect }: Props) => { }, ); + const [pagination, setPagination] = useState(DefaultPagination); + const { pageCount, rowNumber, hasPagination } = usePagination({ + pagination, + setPagination, count: additionalOwners?.filteredCount, - perPage: additionalOwners?.perPage, - page: additionalOwners?.page, - paginate: true, }); const columns = () => [ @@ -96,7 +99,7 @@ const HousingAdditionalOwnerSearchResults = ({ onSelect }: Props) => { /> {hasPagination && (
- dispatch( housingSlice.actions.fetchingAdditionalOwners({ diff --git a/frontend/src/hooks/usePagination.tsx b/frontend/src/hooks/usePagination.tsx index 8002b443b..857c09a78 100644 --- a/frontend/src/hooks/usePagination.tsx +++ b/frontend/src/hooks/usePagination.tsx @@ -2,22 +2,40 @@ import { Pagination } from '@zerologementvacant/models'; import config from '../utils/config'; interface PaginationOptions extends Partial { + pagination: Pagination; count?: number; + setPagination: React.Dispatch>; } export function usePagination(opts: PaginationOptions) { const count = opts.count ?? 0; - const page = opts.page ?? 1; - const perPage = opts.perPage ?? config.perPageDefault; + const page = opts.pagination.page ?? 1; + const perPage = opts.pagination.perPage ?? config.perPageDefault; const pageCount = Math.ceil(count / perPage); const rowNumber = (index: number) => (page - 1) * perPage + index + 1; const hasPagination = count > perPage; + const changePerPage = (perPage: number) => { + opts.setPagination({ + ...opts.pagination, + page: 1, + perPage, + }); + }; + + const changePage = (page: number) => { + opts.setPagination({ + ...opts.pagination, + page, + }); + }; return { pageCount, rowNumber, hasPagination, + changePerPage, + changePage, }; } From 8d3b14ca068b6362dfe8d607d943001f650f473e Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Tue, 30 Jul 2024 10:50:39 +0200 Subject: [PATCH 25/25] feat(server): remove obsolete scripts deduplicate-housing and deduplicate-owners --- .../src/scripts/deduplicate-housing/README.md | 3 - .../deduplicate-housing/housing-stream.ts | 54 ----- .../src/scripts/deduplicate-housing/index.ts | 3 - .../src/scripts/deduplicate-housing/merger.ts | 212 ------------------ .../src/scripts/deduplicate-housing/script.ts | 27 --- .../test/housing-stream.test.ts | 39 ---- .../deduplicate-housing/test/merger.test.ts | 211 ----------------- .../deduplicate-owners/OwnerDuplicate.ts | 8 - .../src/scripts/deduplicate-owners/README.md | 32 --- .../comparisonMergeError.ts | 26 --- .../src/scripts/deduplicate-owners/index.ts | 117 ---------- .../src/scripts/deduplicate-owners/merger.ts | 155 ------------- .../ownersDuplicatesRepository.ts | 49 ---- .../scripts/deduplicate-owners/recorder.ts | 82 ------- .../src/scripts/deduplicate-owners/report.ts | 14 -- .../scripts/deduplicate-owners/reporter.ts | 28 --- .../deduplicate-owners/test/merger.test.ts | 150 ------------- 17 files changed, 1210 deletions(-) delete mode 100644 server/src/scripts/deduplicate-housing/README.md delete mode 100644 server/src/scripts/deduplicate-housing/housing-stream.ts delete mode 100644 server/src/scripts/deduplicate-housing/index.ts delete mode 100644 server/src/scripts/deduplicate-housing/merger.ts delete mode 100644 server/src/scripts/deduplicate-housing/script.ts delete mode 100644 server/src/scripts/deduplicate-housing/test/housing-stream.test.ts delete mode 100644 server/src/scripts/deduplicate-housing/test/merger.test.ts delete mode 100644 server/src/scripts/deduplicate-owners/OwnerDuplicate.ts delete mode 100644 server/src/scripts/deduplicate-owners/README.md delete mode 100644 server/src/scripts/deduplicate-owners/comparisonMergeError.ts delete mode 100644 server/src/scripts/deduplicate-owners/index.ts delete mode 100644 server/src/scripts/deduplicate-owners/merger.ts delete mode 100644 server/src/scripts/deduplicate-owners/ownersDuplicatesRepository.ts delete mode 100644 server/src/scripts/deduplicate-owners/recorder.ts delete mode 100644 server/src/scripts/deduplicate-owners/report.ts delete mode 100644 server/src/scripts/deduplicate-owners/reporter.ts delete mode 100644 server/src/scripts/deduplicate-owners/test/merger.test.ts diff --git a/server/src/scripts/deduplicate-housing/README.md b/server/src/scripts/deduplicate-housing/README.md deleted file mode 100644 index 954e67fab..000000000 --- a/server/src/scripts/deduplicate-housing/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Deprecated - -Can be safely removed. diff --git a/server/src/scripts/deduplicate-housing/housing-stream.ts b/server/src/scripts/deduplicate-housing/housing-stream.ts deleted file mode 100644 index 0dcd2f6fc..000000000 --- a/server/src/scripts/deduplicate-housing/housing-stream.ts +++ /dev/null @@ -1,54 +0,0 @@ -import Stream = Highland.Stream; -import { HousingApi } from '~/models/HousingApi'; -import highland from 'highland'; -import housingRepository, { - Housing, - HousingDBO, - parseHousingApi, -} from '~/repositories/housingRepository'; -import { prependAsync } from '../shared'; - -export function housingStream(): Stream { - const query = Housing() - .whereLike('local_id', '%:%') - .orderBy('local_id') - .stream(); - - return highland(query).map(parseHousingApi).map(validate); -} - -export function parseLocalId(badLocalId: string): string { - const [localId] = badLocalId.split(':'); - return localId; -} - -type HousingByLocalId = Record; - -export function prependOriginalHousing( - stream: Stream, -): Stream { - return stream - .map((group) => Object.values(group)) - .sequence() - .through( - prependAsync(async (housingList) => { - const { geoCode, localId } = housingList[0]; - const originalHousing = await housingRepository.findOne({ - geoCode, - localId: parseLocalId(localId), - }); - return originalHousing ? [originalHousing] : []; - }), - ); -} - -function validate(housing: HousingApi): HousingApi { - return { - ...housing, - mutationDate: - // Specific rule because of the given data - !!housing.mutationDate && housing.mutationDate < new Date('3000-01-01') - ? housing.mutationDate - : null, - }; -} diff --git a/server/src/scripts/deduplicate-housing/index.ts b/server/src/scripts/deduplicate-housing/index.ts deleted file mode 100644 index ba5cac5e0..000000000 --- a/server/src/scripts/deduplicate-housing/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { run } from './script'; - -run(); diff --git a/server/src/scripts/deduplicate-housing/merger.ts b/server/src/scripts/deduplicate-housing/merger.ts deleted file mode 100644 index 7886bf561..000000000 --- a/server/src/scripts/deduplicate-housing/merger.ts +++ /dev/null @@ -1,212 +0,0 @@ -import fp from 'lodash/fp'; -import { HousingApi, HousingRecordApi } from '~/models/HousingApi'; -import { - contramap, - DEFAULT_ORDER, - first, - firstDefined, - max, - merge as mergeObjects, -} from '@zerologementvacant/shared'; -import { logger } from '~/infra/logger'; -import highland from 'highland'; -import { - formatHousingRecordApi, - Housing, -} from '~/repositories/housingRepository'; -import db from '~/infra/database/'; -import { CampaignsHousing } from '~/repositories/campaignHousingRepository'; -import { Knex } from 'knex'; -import { HousingNotes } from '~/repositories/noteRepository'; -import { - GroupHousingEvents, - HousingEvents, -} from '~/repositories/eventRepository'; -import { GroupsHousing } from '~/repositories/groupRepository'; -import Stream = Highland.Stream; - -function merge() { - return (stream: Stream): Stream => { - return stream - .flatMap((housingList) => { - const merged = fp - .orderBy( - ['dataYears', 'mutationDate'], - ['desc', 'desc'], - housingList, - ) - .reduce((a, b) => { - const youngest = youngestOf(a, b); - return mergeObjects({ - id: first, - invariant: first, - localId: first, - buildingId: firstDefined, - buildingGroupId: firstDefined, - rawAddress: first, - geoCode: first, - longitude: firstDefined, - latitude: firstDefined, - cadastralClassification: firstDefined, - uncomfortable: first, - vacancyStartYear: firstDefined, - housingKind: () => youngest.housingKind, - roomsCount: () => youngest.roomsCount, - livingArea: () => youngest.livingArea, - cadastralReference: firstDefined, - buildingYear: firstDefined, - taxed: firstDefined, - vacancyReasons: firstDefined, - dataYears: fp.pipe(fp.union, (dataYears) => - fp.orderBy(fp.identity, 'desc', dataYears), - ), - buildingLocation: firstDefined, - ownershipKind: firstDefined, - status: () => youngest.status, - subStatus: () => youngest.subStatus, - precisions: () => youngest.precisions, - energyConsumption: firstDefined, - energyConsumptionAt: firstDefined, - occupancy: () => youngest.occupancy, - occupancyRegistered: () => youngest.occupancyRegistered, - occupancyIntended: () => youngest.occupancyIntended, - source: () => youngest.source, - mutationDate: firstDefined, - })(a, b); - }); - - return highland(cleanup(merged, housingList)); - }) - .tap((housing) => { - logger.debug('Merged housing', { localId: housing.localId }); - }); - }; -} - -export async function cleanup( - merged: HousingRecordApi, - housingList: HousingRecordApi[], -): Promise { - const houses = housingList.filter((housing) => housing.id !== merged.id); - - await db.transaction(async (transaction) => { - // Transfer campaigns to the merged housing - await CampaignsHousing(transaction).modify( - transfer({ - from: houses, - to: merged, - foreignKey: 'campaign_id', - table: 'campaigns_housing', - }), - ); - await HousingEvents(transaction).modify( - transfer({ - from: houses, - to: merged, - foreignKey: 'event_id', - table: 'housing_events', - }), - ); - await HousingNotes(transaction).modify( - transfer({ - from: houses, - to: merged, - foreignKey: 'note_id', - table: 'housing_notes', - }), - ); - await GroupsHousing(transaction).modify( - transfer({ - from: houses, - to: merged, - foreignKey: 'group_id', - table: 'groups_housing', - }), - ); - await GroupHousingEvents(transaction).modify( - transfer({ - from: houses, - to: merged, - foreignKey: ['event_id', 'group_id'], - table: 'group_housing_events', - }), - ); - - await Housing(transaction) - .insert(formatHousingRecordApi(merged)) - .onConflict(['geo_code', 'local_id']) - .merge(); - - // Remove duplicate houses - await Housing(transaction) - .whereIn( - 'id', - houses.map((housing) => housing.id), - ) - .delete(); - - // Clean up local_id - if (merged.localId.includes(':')) { - await Housing(transaction) - .where({ - geo_code: merged.geoCode, - local_id: merged.localId, - }) - .update({ - local_id: merged.localId.split(':')[0], - }); - } - }); - return merged; -} - -interface TransferOptions { - from: HousingRecordApi[]; - to: HousingRecordApi; - foreignKey: string | string[]; - table: string; -} - -function transfer({ from, to, foreignKey, table }: TransferOptions) { - return (query: Knex.QueryBuilder): void => { - const foreignKeys = Array.isArray(foreignKey) ? foreignKey : [foreignKey]; - query - .update({ - housing_geo_code: to.geoCode, - housing_id: to.id, - }) - .whereIn([...foreignKeys, 'housing_geo_code', 'housing_id'], (query) => { - query - .select([...foreignKeys, 'housing_geo_code', 'housing_id']) - .from(table) - .whereIn( - ['housing_geo_code', 'housing_id'], - from.map((housing) => [housing.geoCode, housing.id]), - ) - .whereNotExists((query) => { - const refs = fp.fromPairs( - foreignKeys.map((key) => [key, db.ref(`${table}.${key}`)]), - ); - query - .select('*') - .from({ subquery: table }) - .where({ - ...refs, - housing_geo_code: to.geoCode, - housing_id: to.id, - }); - }) - .distinctOn(...foreignKeys); - }); - }; -} - -const youngestOf = max( - contramap((housing: HousingRecordApi) => Math.max(...housing.dataYears))( - DEFAULT_ORDER, - ), -); - -export default { - merge, -}; diff --git a/server/src/scripts/deduplicate-housing/script.ts b/server/src/scripts/deduplicate-housing/script.ts deleted file mode 100644 index cd5273707..000000000 --- a/server/src/scripts/deduplicate-housing/script.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { - prependOriginalHousing, - housingStream, - parseLocalId, -} from './housing-stream'; -import merger from './merger'; -import { logger } from '~/infra/logger'; -import { formatElapsed, timer } from '../shared/elapsed'; -import db from '~/infra/database/'; - -export function run() { - const stop = timer(); - - housingStream() - .tap((housing) => { - logger.debug(`Processing ${housing.localId}...`); - }) - .group((housing) => parseLocalId(housing.localId)) - .through(prependOriginalHousing) - .through(merger.merge()) - .done(() => { - const elapsed = stop(); - logger.info(`Done in ${formatElapsed(elapsed)}.`); - - db.destroy(); - }); -} diff --git a/server/src/scripts/deduplicate-housing/test/housing-stream.test.ts b/server/src/scripts/deduplicate-housing/test/housing-stream.test.ts deleted file mode 100644 index 10329f825..000000000 --- a/server/src/scripts/deduplicate-housing/test/housing-stream.test.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { prependOriginalHousing } from '../housing-stream'; -import highland from 'highland'; -import { genHousingApi, genInvariant, genLocalId } from '~/test/testFixtures'; -import { HousingApi } from '~/models/HousingApi'; -import { - formatHousingRecordApi, - Housing, -} from '~/repositories/housingRepository'; - -describe('Housing stream', () => { - describe('concatOriginalHousing', () => { - it('should concat the original housing', async () => { - const localId = genLocalId('12', genInvariant('345')); - const housingList = new Array(3) - .fill('0') - .map(() => genHousingApi('12345')) - .map((housing) => ({ - ...housing, - localId, - })); - const stream = highland( - Promise.resolve({ - [localId]: housingList, - }), - ); - const added: HousingApi = { ...genHousingApi('12345'), localId }; - await Housing().insert(formatHousingRecordApi(added)); - - const actual = await prependOriginalHousing(stream).toPromise(Promise); - - expect(actual).toIncludeAllMembers(housingList); - expect(actual).toPartiallyContain>({ - id: added.id, - invariant: added.invariant, - localId: added.localId, - }); - }); - }); -}); diff --git a/server/src/scripts/deduplicate-housing/test/merger.test.ts b/server/src/scripts/deduplicate-housing/test/merger.test.ts deleted file mode 100644 index f94e40b8b..000000000 --- a/server/src/scripts/deduplicate-housing/test/merger.test.ts +++ /dev/null @@ -1,211 +0,0 @@ -import { - genCampaignApi, - genEstablishmentApi, - genGroupApi, - genGroupHousingEventApi, - genHousingApi, - genHousingEventApi, - genHousingNoteApi, - genUserApi, -} from '~/test/testFixtures'; -import merger, { cleanup } from '../merger'; -import { - Campaigns, - formatCampaignApi, -} from '~/repositories/campaignRepository'; -import { - CampaignHousingDBO, - CampaignsHousing, - formatCampaignHousingApi, -} from '~/repositories/campaignHousingRepository'; -import { - formatHousingRecordApi, - Housing, -} from '~/repositories/housingRepository'; -import { - formatHousingNoteApi, - formatNoteApi, - HousingNotes, - Notes, -} from '~/repositories/noteRepository'; -import { - Events, - formatEventApi, - formatGroupHousingEventApi, - formatHousingEventApi, - GroupHousingEvents, - HousingEvents, -} from '~/repositories/eventRepository'; -import { - formatGroupApi, - formatGroupHousingApi, - Groups, - GroupsHousing, -} from '~/repositories/groupRepository'; -import highland from 'highland'; -import { HousingApi } from '~/models/HousingApi'; -import { - Establishments, - formatEstablishmentApi, -} from '~/repositories/establishmentRepository'; -import { formatUserApi, Users } from '~/repositories/userRepository'; - -describe('Merger', () => { - describe('merge', () => { - function createHousing(data: Partial): HousingApi { - return { ...genHousingApi(), ...data }; - } - - it('should merge the data years', async () => { - const housingList: HousingApi[] = [ - createHousing({ dataYears: [2022, 2021] }), - createHousing({ dataYears: [2022, 2023] }), - ]; - const stream = highland([housingList]); - - const actual = await stream.through(merger.merge()).toPromise(Promise); - - expect(actual).toHaveProperty('dataYears', [2023, 2022, 2021]); - }); - - it('should merge the mutation date', async () => { - const housingList: HousingApi[] = [ - createHousing({ - dataYears: [2022], - mutationDate: new Date('2021-01-01'), - }), - createHousing({ - dataYears: [2022], - mutationDate: null, - }), - createHousing({ - dataYears: [2022], - mutationDate: new Date('2022-01-01'), - }), - ]; - const stream = highland([housingList]); - - const actual = await stream.through(merger.merge()).toPromise(Promise); - - expect(actual).toHaveProperty('mutationDate', new Date('2022-01-01')); - }); - }); - - describe('cleanup', () => { - const establishment = genEstablishmentApi(); - const user = genUserApi(establishment.id); - - beforeAll(async () => { - await Establishments().insert(formatEstablishmentApi(establishment)); - await Users().insert(formatUserApi(user)); - }); - - it('should transfer campaigns to the merged housing', async () => { - const housingList = Array.from({ length: 3 }, () => genHousingApi()); - const [merged] = housingList; - await Housing().insert(housingList.map(formatHousingRecordApi)); - const campaigns = Array.from({ length: 3 }, () => - genCampaignApi(establishment.id, user.id), - ); - await Campaigns().insert(campaigns.map(formatCampaignApi)); - const campaignsHousing: CampaignHousingDBO[] = campaigns - .flatMap((campaign) => - formatCampaignHousingApi(campaign, housingList.slice(1)), - ) - .concat({ - campaign_id: campaigns[0].id, - housing_geo_code: merged.geoCode, - housing_id: merged.id, - }); - await CampaignsHousing().insert(campaignsHousing); - - await cleanup(merged, housingList); - - const actual = await CampaignsHousing().where({ - housing_geo_code: merged.geoCode, - housing_id: merged.id, - }); - expect(actual).toBeArrayOfSize(campaigns.length); - }); - - it('should transfer events to the merged housing', async () => { - const housingList = Array.from({ length: 3 }, () => genHousingApi()); - const [merged] = housingList; - await Housing().insert(housingList.map(formatHousingRecordApi)); - const events = housingList.map((housing) => - genHousingEventApi(housing, user), - ); - await Events().insert(events.map(formatEventApi)); - await HousingEvents().insert(events.map(formatHousingEventApi)); - - await cleanup(merged, housingList); - - const actual = await HousingEvents().where({ - housing_geo_code: merged.geoCode, - housing_id: merged.id, - }); - expect(actual).toBeArrayOfSize(events.length); - }); - - it('should transfer notes to the merged housing', async () => { - const housingList = Array.from({ length: 3 }, () => genHousingApi()); - const [merged] = housingList; - await Housing().insert(housingList.map(formatHousingRecordApi)); - const notes = housingList.map((housing) => - genHousingNoteApi(user, housing), - ); - await Notes().insert(notes.map(formatNoteApi)); - await HousingNotes().insert(notes.map(formatHousingNoteApi)); - - await cleanup(merged, housingList); - - const actual = await HousingNotes().where({ - housing_geo_code: merged.geoCode, - housing_id: merged.id, - }); - expect(actual).toBeArrayOfSize(notes.length); - }); - - it('should transfer groups to the merged housing', async () => { - const housingList = Array.from({ length: 3 }, () => genHousingApi()); - const [merged] = housingList; - await Housing().insert(housingList.map(formatHousingRecordApi)); - const groups = housingList.map(() => genGroupApi(user, establishment)); - await Groups().insert(groups.map(formatGroupApi)); - await GroupsHousing().insert( - groups.flatMap((group) => formatGroupHousingApi(group, housingList)), - ); - - await cleanup(merged, housingList); - - const actual = await GroupsHousing().where({ - housing_geo_code: merged.geoCode, - housing_id: merged.id, - }); - expect(actual).toBeArrayOfSize(groups.length); - }); - - it('should transfer group events to the merged housing', async () => { - const housingList = Array.from({ length: 3 }, () => genHousingApi()); - const [merged] = housingList; - await Housing().insert(housingList.map(formatHousingRecordApi)); - const groups = housingList.map(() => genGroupApi(user, establishment)); - await Groups().insert(groups.map(formatGroupApi)); - const events = housingList.flatMap((housing) => { - return groups.map((group) => - genGroupHousingEventApi(housing, group, user), - ); - }); - await Events().insert(events.map(formatEventApi)); - await GroupHousingEvents().insert(events.map(formatGroupHousingEventApi)); - - await cleanup(merged, housingList); - - const actual = await GroupHousingEvents().where({ - housing_geo_code: merged.geoCode, - housing_id: merged.id, - }); - expect(actual).toBeArrayOfSize(events.length); - }); - }); -}); diff --git a/server/src/scripts/deduplicate-owners/OwnerDuplicate.ts b/server/src/scripts/deduplicate-owners/OwnerDuplicate.ts deleted file mode 100644 index 2f7e5769a..000000000 --- a/server/src/scripts/deduplicate-owners/OwnerDuplicate.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { OwnerApi } from '~/models/OwnerApi'; - -export interface OwnerDuplicate extends OwnerApi { - /** - * The source owner that was compared with this duplicate. - */ - sourceId: string; -} diff --git a/server/src/scripts/deduplicate-owners/README.md b/server/src/scripts/deduplicate-owners/README.md deleted file mode 100644 index 6999dbbc2..000000000 --- a/server/src/scripts/deduplicate-owners/README.md +++ /dev/null @@ -1,32 +0,0 @@ -# Deduplicate owners - -## Why - -We have duplicate owners in our database that should be merged to form -a single national owner. - -## What - -A script that deduplicates owners, run manually or as a cron job. - -## How - -A stream of our owners grouped by name is set up. -For each owner, find all owners with the same name. -For each list of owners with the same name: -- for each owner, compare it to all *other* owners in the list - -We get a stream of comparisons that is forked into several streams: -- a stream recording the comparisons and logging stats about them -- a stream keeping and saving duplicates that need human review -- a stream keeping matches that can be merged automatically - -Before merging, events, notes and attached housing are transferred to the owner -we keep. The duplicates get merged in this owner and it is saved to the -database, ending the transaction. - -The reporter writes a report to the standard output. - -## When - -Whenever needed. diff --git a/server/src/scripts/deduplicate-owners/comparisonMergeError.ts b/server/src/scripts/deduplicate-owners/comparisonMergeError.ts deleted file mode 100644 index 8f741bfa0..000000000 --- a/server/src/scripts/deduplicate-owners/comparisonMergeError.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { Comparison } from '../shared'; - -interface ConstructorOptions { - comparison: Comparison; - origin: Error; -} - -export class ComparisonMergeError extends Error { - private readonly comparison: Comparison; - private readonly origin: Error; - - constructor(opts: ConstructorOptions) { - super('Comparison error'); - this.comparison = opts.comparison; - this.origin = opts.origin; - } - - toJSON() { - return { - message: this.message, - name: this.message, - comparison: this.comparison, - origin: this.origin, - }; - } -} diff --git a/server/src/scripts/deduplicate-owners/index.ts b/server/src/scripts/deduplicate-owners/index.ts deleted file mode 100644 index 5c25f2f17..000000000 --- a/server/src/scripts/deduplicate-owners/index.ts +++ /dev/null @@ -1,117 +0,0 @@ -import highland from 'highland'; -import script from 'node:process'; - -import { logger } from '~/infra/logger'; -import ownerRepository from '~/repositories/ownerRepository'; -import { - isMatch, - needsManualReview, -} from '../shared/owner-processor/duplicates'; -import db from '~/infra/database/'; -import { createReporter } from './reporter'; -import { createRecorder } from './recorder'; -import merger from './merger'; -import ownersDuplicatesRepository from './ownersDuplicatesRepository'; -import { OwnerDuplicate } from './OwnerDuplicate'; -import { formatElapsed, timer } from '../shared/elapsed'; -import evaluator from '../shared/owner-processor/evaluator'; - -const recorder = createRecorder(); -const reporter = createReporter('json'); - -function run(): void { - const comparisons = ownerRepository - .stream({ - groupBy: ['full_name'], - }) - .tap((owner) => logger.debug(`Processing ${owner.fullName}...`)) - .through(evaluator.evaluate()); - - comparisons - .fork() - .through(recorder.record()) - .map(reporter.toString) - .each((report) => { - logger.info(report); - }); - - const duplicateWriter = comparisons - .fork() - .map((comparison) => - comparison.duplicates - .filter((duplicate) => - needsManualReview(comparison.source, [duplicate]), - ) - .map((duplicate) => ({ - ...duplicate.value, - sourceId: comparison.source.id, - })), - ) - .flatten() - .tap((duplicate) => - logger.trace('Found duplicate', { - id: duplicate.id, - fullName: duplicate.fullName, - }), - ) - .batch(1000) - .flatMap((duplicates) => - highland(ownersDuplicatesRepository.save(...duplicates)), - ); - - const ownerMerger = comparisons - .fork() - .filter((comparison) => isMatch(comparison.score)) - .filter((comparison) => !comparison.needsReview) - .on('owners:removed', (count: number) => { - recorder.update({ - removed: { - owners: recorder.report.removed.owners + count, - }, - }); - }) - .on('owners-housing:removed', (count: number) => { - recorder.update({ - removed: { - ownersHousing: recorder.report.removed.ownersHousing + count, - }, - }); - }) - .through(merger.merge()); - - highland([duplicateWriter, ownerMerger]) - .merge() - .stopOnError((error) => { - logger.error(error); - }) - .done(() => { - logger.info('Duplicates written.'); - logger.info('Owners merged.'); - - ownersDuplicatesRepository.removeOrphans().then(() => { - script.exit(); - }); - }); -} - -script.once('SIGINT', cleanUp); -script.once('SIGTERM', cleanUp); - -function cleanUp() { - recorder.flush(); - const report = reporter.toString(recorder.report); - logger.info(report); - logger.info('Report written.'); - script.exit(); -} - -const stop = timer(); - -script.on('exit', () => { - const elapsed = formatElapsed(stop()); - logger.info(`Done in ${elapsed}.`); - - return db.destroy(); -}); - -run(); diff --git a/server/src/scripts/deduplicate-owners/merger.ts b/server/src/scripts/deduplicate-owners/merger.ts deleted file mode 100644 index 9df5d05f1..000000000 --- a/server/src/scripts/deduplicate-owners/merger.ts +++ /dev/null @@ -1,155 +0,0 @@ -import { Comparison } from '../shared/models/Comparison'; -import { logger } from '~/infra/logger'; -import { ownerNotesTable } from '~/repositories/noteRepository'; -import { ownerEventsTable } from '~/repositories/eventRepository'; -import { OwnerDBO, Owners, ownerTable } from '~/repositories/ownerRepository'; -import db from '~/infra/database/'; -import { - isMatch, - isStreetNumber, - needsManualReview, -} from '../shared/owner-processor/duplicates'; -import fp from 'lodash/fp'; -import { ComparisonMergeError } from './comparisonMergeError'; -import { - HousingOwnerDBO, - housingOwnersTable, -} from '~/repositories/housingOwnerRepository'; -import Stream = Highland.Stream; -import highland from 'highland'; - -export async function merge( - comparison: Comparison, - stream?: Stream, -): Promise { - if (comparison.needsReview || (await wasRemoved(comparison.source.id))) { - return; - } - - const keeping = comparison.source; - const removing = comparison.duplicates - .filter( - (owner) => - isMatch(owner.score) && !needsManualReview(comparison.source, [owner]), - ) - .map((owner) => owner.value); - - const removingIds = removing.map((owner) => owner.id); - // Add removed ids - await db - .transaction(async (transaction) => { - // Handle the case when the owner appears multiple times in a housing - const ownersHousing = await transaction(housingOwnersTable).whereIn( - 'owner_id', - [...removingIds, keeping.id], - ); - const duplicates = findHousingOwnerDuplicates(ownersHousing); - if (duplicates.length > 0) { - const duplicatesRemoved = await transaction(housingOwnersTable) - .modify((query) => { - duplicates.forEach((duplicate) => { - query.orWhere({ - housing_id: duplicate.housing_id, - housing_geo_code: duplicate.housing_geo_code, - owner_id: duplicate.owner_id, - }); - }); - }) - .delete(); - stream?.emit('owners-housing:removed', duplicatesRemoved); - logger.debug( - `Removed ${duplicatesRemoved} duplicate(s) from owners_housing`, - ); - } - - await Promise.all([ - // Transfer housing owners - transaction(housingOwnersTable) - .update({ owner_id: keeping.id }) - .whereIn('owner_id', removingIds), - // Transfer owner events - transaction(ownerEventsTable) - .update({ owner_id: keeping.id }) - .whereIn('owner_id', removingIds), - // Transfer owner notes - transaction(ownerNotesTable) - .update({ owner_id: keeping.id }) - .whereIn('owner_id', removingIds), - // Transfer old events - transaction('old_events') - .update({ owner_id: keeping.id }) - .whereIn('owner_id', removingIds), - ]); - - const removed = await transaction(ownerTable) - .whereIn('id', removingIds) - .delete(); - stream?.emit('owners:removed', removed); - logger.info('Removed owners', { - fullName: keeping.fullName, - removed, - }); - - const owners = [keeping, ...removing]; - const merged: Partial = fp.pickBy((value) => !fp.isNil(value), { - raw_address: fp - .maxBy((owner) => owner.rawAddress.length, owners) - ?.rawAddress?.map((address) => address.replace(/\s+/g, ' ')) - ?.map((address) => - isStreetNumber(address) ? fp.trimCharsStart('0', address) : address, - ), - birth_date: !keeping.birthDate - ? owners.find((owner) => !!owner.birthDate)?.birthDate - : undefined, - administrator: owners.find((owner) => !!owner.administrator) - ?.administrator, - email: owners.find((owner) => !!owner.email)?.email, - phone: owners.find((owner) => !!owner.phone)?.phone, - owner_kind: owners.find((owner) => !!owner.kind)?.kind, - owner_kind_detail: owners.find((owner) => !!owner.kindDetail) - ?.kindDetail, - }); - if (fp.size(merged) > 0) { - logger.debug('Merge into owner', { id: keeping.id, ...merged }); - await transaction(ownerTable).where({ id: keeping.id }).update(merged); - } - }) - .catch((error) => { - throw new ComparisonMergeError({ comparison, origin: error }); - }); -} - -async function wasRemoved(id: string): Promise { - const owner = await Owners().where('id', id).first(); - return !owner; -} - -function findHousingOwnerDuplicates( - housingOwners: HousingOwnerDBO[], -): HousingOwnerDBO[] { - return fp.pipe( - fp.groupBy('housing_id'), - fp.pickBy((housingOwners) => housingOwners.length > 1), - fp.mapValues((housingOwners) => { - const min = fp.minBy('rank', housingOwners); - if (!min) { - throw new Error('There should be housing owners here'); - } - return housingOwners.filter( - (housingOwner) => housingOwner.owner_id !== min.owner_id, - ); - }), - fp.values, - fp.flatten, - )(housingOwners); -} - -export default { - merge() { - return (stream: Stream): Stream => { - return stream.flatMap((comparison) => - highland(merge(comparison, stream)), - ); - }; - }, -}; diff --git a/server/src/scripts/deduplicate-owners/ownersDuplicatesRepository.ts b/server/src/scripts/deduplicate-owners/ownersDuplicatesRepository.ts deleted file mode 100644 index 9d8835a41..000000000 --- a/server/src/scripts/deduplicate-owners/ownersDuplicatesRepository.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { OwnerDuplicate } from './OwnerDuplicate'; -import { - formatOwnerApi, - OwnerDBO, - ownerTable, -} from '~/repositories/ownerRepository'; -import db from '~/infra/database/'; -import { logger } from '~/infra/logger'; - -const ownerDuplicatesTable = 'owners_duplicates'; - -async function save(...duplicates: OwnerDuplicate[]): Promise { - const entities = duplicates.map(formatOwnerDuplicate); - logger.debug(`Saving ${entities.length} owner duplicates...`); - await db(ownerDuplicatesTable).insert(entities).onConflict('id').ignore(); -} - -async function removeOrphans(): Promise { - logger.debug('Removing orphan duplicates...'); - const deleted = await db(ownerDuplicatesTable) - .whereNotExists((builder) => - builder - .select('id') - .from(ownerTable) - .where('id', db.column(`${ownerDuplicatesTable}.source_id`)), - ) - .delete(); - logger.debug(`Removed ${deleted} orphan duplicates.`); -} - -export interface OwnerDuplicateDBO extends OwnerDBO { - source_id: string; -} - -export function formatOwnerDuplicate( - duplicate: OwnerDuplicate, -): OwnerDuplicateDBO { - return { - ...formatOwnerApi(duplicate), - source_id: duplicate.sourceId, - }; -} - -const ownersDuplicatesRepository = { - save, - removeOrphans, -}; - -export default ownersDuplicatesRepository; diff --git a/server/src/scripts/deduplicate-owners/recorder.ts b/server/src/scripts/deduplicate-owners/recorder.ts deleted file mode 100644 index ed9ed00f3..000000000 --- a/server/src/scripts/deduplicate-owners/recorder.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { Comparison } from '../shared/'; -import { Report } from './report'; -import { logger } from '~/infra/logger'; -import { isMatch } from '../shared/owner-processor/duplicates'; -import { DeepPartial } from 'ts-essentials'; -import Stream = Highland.Stream; - -class Recorder { - report: Report = { - overall: 0, - match: 0, - nonMatch: 0, - needReview: 0, - removed: { - owners: 0, - ownersHousing: 0, - }, - score: { - sum: 0, - mean: 0, - }, - }; - - record() { - return (stream: Stream): Stream => { - return stream - .reduce(this.report, (acc, comparison) => { - const match = isMatch(comparison.score) && !comparison.needsReview; - const nonMatch = - !isMatch(comparison.score) && !comparison.needsReview; - - this.report = { - overall: acc.overall + 1, - match: acc.match + (match ? 1 : 0), - nonMatch: acc.nonMatch + (nonMatch ? 1 : 0), - needReview: acc.needReview + (comparison.needsReview ? 1 : 0), - removed: this.report.removed, - score: { - sum: acc.score.sum + comparison.score, - mean: 0, - }, - }; - return this.report; - }) - .map(computeScores); - }; - } - - update(report: DeepPartial): void { - this.report = { - ...this.report, - ...report, - removed: { - ...this.report.removed, - ...report.removed, - }, - score: { - ...this.report.score, - ...report.score, - }, - }; - } - - flush(): void { - this.report = computeScores(this.report); - logger.debug('Scores computed', this.report.score); - } -} - -function computeScores(report: Report): Report { - return { - ...report, - score: { - ...report.score, - mean: report.score.sum / report.overall, - }, - }; -} - -export function createRecorder() { - return new Recorder(); -} diff --git a/server/src/scripts/deduplicate-owners/report.ts b/server/src/scripts/deduplicate-owners/report.ts deleted file mode 100644 index cdfab5701..000000000 --- a/server/src/scripts/deduplicate-owners/report.ts +++ /dev/null @@ -1,14 +0,0 @@ -export interface Report { - overall: number; - match: number; - nonMatch: number; - needReview: number; - removed: { - owners: number; - ownersHousing: number; - }; - score: { - sum: number; - mean: number; - }; -} diff --git a/server/src/scripts/deduplicate-owners/reporter.ts b/server/src/scripts/deduplicate-owners/reporter.ts deleted file mode 100644 index 3ad304eef..000000000 --- a/server/src/scripts/deduplicate-owners/reporter.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { Report } from './report'; - -interface Reporter { - toString(report: Report): string; -} - -class TextReporter implements Reporter { - toString(report: Report): string { - return Object.keys(report) - .map((key) => `${key}: ${report[key as keyof Report]}`) - .join('\n'); - } -} - -class JSONReporter implements Reporter { - toString(report: Report): string { - return JSON.stringify(report); - } -} - -export function createReporter(type: 'text' | 'json'): Reporter { - switch (type) { - case 'text': - return new TextReporter(); - case 'json': - return new JSONReporter(); - } -} diff --git a/server/src/scripts/deduplicate-owners/test/merger.test.ts b/server/src/scripts/deduplicate-owners/test/merger.test.ts deleted file mode 100644 index a56f7249b..000000000 --- a/server/src/scripts/deduplicate-owners/test/merger.test.ts +++ /dev/null @@ -1,150 +0,0 @@ -import { merge } from '../merger'; -import { Comparison } from '../../shared'; -import { - genEstablishmentApi, - genHousingApi, - genOwnerApi, - genOwnerEventApi, - genUserApi, -} from '~/test/testFixtures'; -import db from '~/infra/database/'; -import { formatOwnerApi, ownerTable } from '~/repositories/ownerRepository'; -import { - eventsTable, - formatEventApi, - ownerEventsTable, -} from '~/repositories/eventRepository'; -import { - formatHousingRecordApi, - housingTable, -} from '~/repositories/housingRepository'; -import { - HousingOwnerDBO, - housingOwnersTable, -} from '~/repositories/housingOwnerRepository'; -import { - Establishments, - formatEstablishmentApi, -} from '~/repositories/establishmentRepository'; -import { formatUserApi, Users } from '~/repositories/userRepository'; - -describe('Merger', () => { - const establishment = genEstablishmentApi(); - const user = genUserApi(establishment.id); - - beforeAll(async () => { - await Establishments().insert(formatEstablishmentApi(establishment)); - await Users().insert(formatUserApi(user)); - }); - - describe('merge', () => { - const source = genOwnerApi(); - const duplicates = Array.from({ length: 2 }, () => genOwnerApi()).map( - (owner) => ({ ...owner, birthDate: undefined }), - ); - const suggestion = source; - const events = duplicates.map((owner) => - genOwnerEventApi(owner.id, user.id), - ); - const housingList = duplicates.map(() => genHousingApi()); - - beforeAll(async () => { - const owners = [source, ...duplicates].map(formatOwnerApi); - await db(ownerTable).insert(owners); - - // Attach events to the duplicates - await db(eventsTable).insert(events.map(formatEventApi)); - await db(ownerEventsTable).insert( - events.map((event) => ({ - owner_id: event.old?.id, - event_id: event.id, - })), - ); - - // Attach housing to the duplicates - await db(housingTable).insert(housingList.map(formatHousingRecordApi)); - await db(housingOwnersTable).insert( - housingList.map((housing, i) => { - return { - housing_id: housing.id, - housing_geo_code: housing.geoCode, - owner_id: duplicates[i].id, - rank: i + 1, - }; - }), - ); - }); - - it('should abort if the comparison needs human review', async () => { - const comparison: Comparison = { - score: 0.8, - source, - duplicates: [ - { - score: 0.8, - value: duplicates[0], - }, - ], - needsReview: true, - }; - - await merge(comparison); - - const ownerEvents = await db(ownerEventsTable).whereIn( - 'event_id', - events.map((event) => event.id), - ); - expect(ownerEvents).toSatisfyAll( - (event) => event.owner_id !== suggestion.id, - ); - }); - - it('should transfer housing, events and notes to the owner we keep', async () => { - const comparison: Comparison = { - score: 1, - source, - duplicates: duplicates.map((duplicate) => ({ - score: 1, - value: duplicate, - })), - needsReview: false, - }; - - await merge(comparison); - - const ownerHousing = await db(housingOwnersTable).whereIn( - 'housing_id', - housingList.map((housing) => housing.id), - ); - expect(ownerHousing).toSatisfyAll((oh) => oh.owner_id === suggestion.id); - - const ownerEvents = await db(ownerEventsTable).whereIn( - 'event_id', - events.map((event) => event.id), - ); - expect(ownerEvents).toSatisfyAll( - (event) => event.owner_id === suggestion.id, - ); - }); - - it('should remove duplicates afterwards', async () => { - const comparison: Comparison = { - score: 1, - source, - duplicates: duplicates.map((duplicate) => ({ - score: 1, - value: duplicate, - })), - needsReview: false, - }; - - await merge(comparison); - - const dups = await db(ownerTable).whereIn( - 'id', - duplicates.map((duplicate) => duplicate.id), - ); - expect(dups).toBeArrayOfSize(0); - }); - }); -});