fix(lbac-2173): dedoublonnage: revue marion (#1845)

* fix(lbac-2173): dedoublonnage: revue marion

* fix: rank a chaque passage

* fix: update snapshot

server/src/jobs/offrePartenaire/detectDuplicateJobPartners.ts

@@ -3,8 +3,9 @@ import { AnyBulkWriteOperation, Filter } from "mongodb"
 import { oleoduc, writeData } from "oleoduc"
 import { RECRUITER_STATUS } from "shared/constants"
 import { IJob, JOB_STATUS, JOB_STATUS_ENGLISH, ZGlobalAddress } from "shared/models"
+import { IComputedJobPartnersDuplicateRef } from "shared/models/jobPartnersDuplicateRef"
 import jobsPartnersModel, { IJobsPartnersOfferPrivate } from "shared/models/jobsPartners.model"
-import jobsPartnersComputedModel, { IComputedJobPartnersDuplicateRef, IComputedJobsPartners, JOB_PARTNER_BUSINESS_ERROR } from "shared/models/jobsPartnersComputed.model"
+import jobsPartnersComputedModel, { IComputedJobsPartners, JOB_PARTNER_BUSINESS_ERROR } from "shared/models/jobsPartnersComputed.model"
 import recruiterModel, { IRecruiter } from "shared/models/recruiter.model"
 import { removeAccents } from "shared/utils"
 import * as stringSimilarity from "string-similarity"

server/src/jobs/offrePartenaire/fillFieldsForPartnersFactory.ts

@@ -1,5 +1,5 @@
 import { internal } from "@hapi/boom"
-import { Filter } from "mongodb"
+import { AnyBulkWriteOperation, Filter } from "mongodb"
 import { oleoduc, writeData } from "oleoduc"
 import { IJobsPartnersOfferPrivate } from "shared/models/jobsPartners.model"
 import { COMPUTED_ERROR_SOURCE, IComputedJobsPartners } from "shared/models/jobsPartnersComputed.model"
@@ -62,12 +62,14 @@ export const fillFieldsForPartnersFactory = async <SourceFields extends keyof IJ
   const toUpdateCount = await getDbCollection("computed_jobs_partners").countDocuments(queryFilter)
   logger.info(`${toUpdateCount} documents à traiter`)
   const counters = { total: 0, success: 0, error: 0 }
+  const now = new Date()
   await oleoduc(
     getDbCollection("computed_jobs_partners")
       .find(queryFilter, {
         projection: {
           ...Object.fromEntries([...sourceFields, ...filledFields].map((field) => [field, 1])),
           _id: 1,
+          jobs_in_success: 1,
         },
       })
       .stream(),
@@ -79,16 +81,19 @@ export const fillFieldsForPartnersFactory = async <SourceFields extends keyof IJ
         try {
           const responses = await getData(documents)
 
-          const dataToWrite = responses.flatMap((document) => {
-            const { _id, ...newFields } = document
-            return [
+          const dataToWrite = responses.flatMap((response) => {
+            const { _id, ...newFields } = response
+            const updates: AnyBulkWriteOperation<IComputedJobsPartners>[] = [
               {
                 updateOne: {
                   filter: { _id },
-                  update: { $set: newFields },
+                  update: { $set: { ...newFields, updated_at: now } },
                 },
               },
-              {
+            ]
+            const document = documents.find((doc2) => doc2._id.equals(_id))
+            if (!document?.jobs_in_success.includes(job)) {
+              updates.push({
                 updateOne: {
                   filter: { _id },
                   update: {
@@ -97,8 +102,9 @@ export const fillFieldsForPartnersFactory = async <SourceFields extends keyof IJ
                     },
                   },
                 },
-              },
-            ]
+              })
+            }
+            return updates
           })
           if (dataToWrite.length) {
             await getDbCollection("computed_jobs_partners").bulkWrite(dataToWrite, {

server/src/jobs/offrePartenaire/hellowork/__snapshots__/importHelloWork.test.ts.snap

@@ -44,7 +44,7 @@ Vous contribuez au développement commercial en concevant des nouvelles offres,
     "offer_to_be_acquired_skills": [],
     "partner_job_id": "68087320",
     "partner_label": "Hellowork",
-    "updated_at": 2024-02-25T04:36:25.000Z,
+    "updated_at": 2024-07-21T02:49:06.000Z,
     "validated": false,
     "workplace_address_city": "VINCENNES",
     "workplace_address_label": "VINCENNES 94300",
@@ -142,7 +142,7 @@ Rejoignez-nous et venez goûter à une expérience unique !",
     "offer_to_be_acquired_skills": [],
     "partner_job_id": "68090104",
     "partner_label": "Hellowork",
-    "updated_at": 2024-02-25T04:36:28.000Z,
+    "updated_at": 2024-07-21T02:49:06.000Z,
     "validated": false,
     "workplace_address_city": "LAVAL",
     "workplace_address_label": "LAVAL 53000",
@@ -240,7 +240,7 @@ En tant que leader, vous dirigerez les équipes marketing et communication, coor
     "offer_to_be_acquired_skills": [],
     "partner_job_id": "68110296",
     "partner_label": "Hellowork",
-    "updated_at": 2024-02-25T04:36:28.000Z,
+    "updated_at": 2024-07-21T02:49:06.000Z,
     "validated": false,
     "workplace_address_city": "TINQUEUX",
     "workplace_address_label": "TINQUEUX 51430",
@@ -330,7 +330,7 @@ Chez Exotec, nous garantissons l'égalité des chances dans notre processus de r
     "offer_to_be_acquired_skills": [],
     "partner_job_id": "68111676",
     "partner_label": "Hellowork",
-    "updated_at": 2024-02-25T04:36:28.000Z,
+    "updated_at": 2024-07-21T02:49:06.000Z,
     "validated": false,
     "workplace_address_city": "LILLE",
     "workplace_address_label": "LILLE 59160",
@@ -422,7 +422,7 @@ Ref : 1jvie4rokd",
     "offer_to_be_acquired_skills": [],
     "partner_job_id": "68113693",
     "partner_label": "Hellowork",
-    "updated_at": 2024-02-25T04:36:31.000Z,
+    "updated_at": 2024-07-21T02:49:06.000Z,
     "validated": false,
     "workplace_address_city": "ECHIROLLES",
     "workplace_address_label": "ECHIROLLES 38130",

server/src/jobs/offrePartenaire/importFromComputedToJobsPartners.ts

@@ -69,6 +69,7 @@ export const importFromComputedToJobsPartners = async (addedMatchFilter?: Filter
           offer_multicast: computedJobPartner.offer_multicast ?? true,
           offer_origin: computedJobPartner.offer_origin ?? null,
           rank: computedJobPartner.rank ?? null,
+          duplicates: computedJobPartner.duplicates ?? null,
         }
 
         await getDbCollection("jobs_partners").updateOne(

server/src/jobs/offrePartenaire/processJobPartnersForApi.ts

@@ -1,13 +1,20 @@
+import { ObjectId } from "mongodb"
+
 import { getDbCollection } from "@/common/utils/mongodbUtils"
 
 import { fillComputedJobsPartners } from "./fillComputedJobsPartners"
 import { importFromComputedToJobsPartners } from "./importFromComputedToJobsPartners"
 import { jobPartnersByFlux } from "./processJobPartners"
 
 export const processJobPartnersForApi = async () => {
-  const filter = { partner_label: { $nin: jobPartnersByFlux } }
-
+  const processId: string = new ObjectId().toString()
+  await getDbCollection("computed_jobs_partners").updateMany(
+    { partner_label: { $nin: jobPartnersByFlux }, currently_processed_id: null },
+    { $set: { currently_processed_id: processId } }
+  )
+  const filter = { currently_processed_id: processId }
   await fillComputedJobsPartners(filter)
   await importFromComputedToJobsPartners(filter)
   await getDbCollection("computed_jobs_partners").deleteMany({ $and: [filter, { validated: true }] })
+  await getDbCollection("computed_jobs_partners").updateMany(filter, { $set: { currently_processed_id: null } })
 }

server/src/jobs/offrePartenaire/rawToComputedJobsPartners.ts

@@ -46,6 +46,7 @@ export const rawToComputedJobsPartners = async <ZodInput extends AnyZodObject>({
             ...computedJobPartner,
             partner_label: partnerLabel,
             created_at: importDate,
+            updated_at: importDate,
             offer_status_history: [],
           })
           counters.success++

server/src/jobs/offrePartenaire/validateComputedJobPartners.ts

@@ -1,7 +1,7 @@
 import { AnyBulkWriteOperation, Filter } from "mongodb"
 import { oleoduc, writeData } from "oleoduc"
 import jobsPartnersModel from "shared/models/jobsPartners.model"
-import { COMPUTED_ERROR_SOURCE, IComputedJobsPartners, JOB_PARTNER_BUSINESS_ERROR } from "shared/models/jobsPartnersComputed.model"
+import { COMPUTED_ERROR_SOURCE, IComputedJobsPartners } from "shared/models/jobsPartnersComputed.model"
 
 import { logger } from "@/common/logger"
 import { getDbCollection } from "@/common/utils/mongodbUtils"
@@ -55,16 +55,6 @@ export const validateComputedJobPartners = async (addedMatchFilter?: Filter<ICom
                 },
               },
             })
-            operations.push({
-              updateOne: {
-                filter: { _id: document._id },
-                update: {
-                  $set: {
-                    business_error: JOB_PARTNER_BUSINESS_ERROR.ZOD_VALIDATION,
-                  },
-                },
-              },
-            })
           } else {
             counters.success++
           }

shared/models/jobPartnersDuplicateRef.ts

@@ -0,0 +1,13 @@
+import { z } from "zod"
+
+import { zObjectId } from "./common.js"
+import recruiterModel from "./recruiter.model.js"
+
+export const ZComputedJobPartnersDuplicateRef = z.object({
+  otherOfferId: zObjectId,
+  // hardcoded collection names to avoid cyclic dependancies
+  collectionName: z.enum([recruiterModel.collectionName, "jobs_partners", "computed_jobs_partners"]).describe("nom de la collection contenant l'offre avec _id=otherOfferId"),
+  reason: z.string(),
+})
+
+export type IComputedJobPartnersDuplicateRef = z.output<typeof ZComputedJobPartnersDuplicateRef>

shared/models/jobsPartners.model.ts

@@ -6,6 +6,7 @@ import { extensions } from "../helpers/zodHelpers/zodPrimitives.js"
 import { ZPointGeometry } from "./address.model.js"
 import { IModelDescriptor, zObjectId } from "./common.js"
 import { JOB_STATUS_ENGLISH } from "./job.model.js"
+import { ZComputedJobPartnersDuplicateRef } from "./jobPartnersDuplicateRef.js"
 import { zOpcoLabel } from "./opco.model.js"
 
 const collectionName = "jobs_partners" as const
@@ -111,6 +112,7 @@ export const ZJobsPartnersOfferPrivate = ZJobsPartnersOfferApi.omit({
     _id: zObjectId,
     apply_url: ZJobsPartnersOfferApi.shape.apply_url.nullable().default(null),
     rank: z.number().nullish().describe("Valeur indiquant la qualité de l'offre. Plus la valeur est élevée, plus la qualité de l'offre est importante"),
+    duplicates: z.array(ZComputedJobPartnersDuplicateRef).nullish().describe("Référence les autres offres en duplicata avec celle-ci"),
   })
 
 export const ZJobsPartnersOfferPrivateWithDistance = ZJobsPartnersOfferPrivate.extend({

shared/models/jobsPartnersComputed.model.ts

@@ -4,10 +4,7 @@ import { IModelDescriptor, zObjectId } from "shared/models/common.js"
 
 import { extensions } from "../helpers/zodHelpers/zodPrimitives.js"
 
-import jobsPartnersModel, { ZJobsPartnersOfferPrivate } from "./jobsPartners.model.js"
-import recruiterModel from "./recruiter.model.js"
-
-const collectionName = "computed_jobs_partners" as const
+import { ZJobsPartnersOfferPrivate } from "./jobsPartners.model.js"
 
 export enum COMPUTED_ERROR_SOURCE {
   API_SIRET = "api_siret",
@@ -21,22 +18,11 @@ export enum COMPUTED_ERROR_SOURCE {
 export enum JOB_PARTNER_BUSINESS_ERROR {
   CLOSED_COMPANY = "CLOSED_COMPANY",
   DUPLICATE = "DUPLICATE",
-  ZOD_VALIDATION = "ZOD_VALIDATION",
   STAGE = "STAGE",
   EXPIRED = "EXPIRED",
   CFA = "CFA",
 }
 
-export const ZComputedJobPartnersDuplicateRef = z.object({
-  otherOfferId: zObjectId,
-  collectionName: z
-    .enum([recruiterModel.collectionName, collectionName, jobsPartnersModel.collectionName])
-    .describe("nom de la collection contenant l'offre avec _id=otherOfferId"),
-  reason: z.string(),
-})
-
-export type IComputedJobPartnersDuplicateRef = z.output<typeof ZComputedJobPartnersDuplicateRef>
-
 export const ZComputedJobsPartners = extensions
   .optionalToNullish(ZJobsPartnersOfferPrivate.partial())
   .omit({
@@ -60,7 +46,10 @@ export const ZComputedJobsPartners = extensions
     ),
     validated: z.boolean().default(false).describe("Toutes les données nécessaires au passage vers jobs_partners sont présentes et valides (validation zod)"),
     business_error: z.string().nullable().default(null),
-    duplicates: z.array(ZComputedJobPartnersDuplicateRef).nullish().describe("Référence les autres offres en duplicata avec celle-ci"),
+    currently_processed_id: z
+      .string()
+      .nullish()
+      .describe("Si le champ est rempli, l'offre est en train d'être traitée. Les offres ayant le même id sont traitées dans le même batch"),
   })
 export type IComputedJobsPartners = z.output<typeof ZComputedJobsPartners>
 
@@ -69,12 +58,23 @@ export default {
   indexes: [
     [{ partner_job_id: 1 }, {}],
     [{ partner_label: 1 }, {}],
-    [{ validated: 1 }, {}],
-    [{ errors: 1 }, {}],
     [{ partner_label: 1, partner_job_id: 1 }, { unique: true }],
-    [{ workplace_siret: 1 }, {}],
+
+    [{ created_at: 1 }, {}],
+    [{ updated_at: 1 }, {}],
+    [{ business_error: 1 }, {}],
+    [{ errors: 1 }, {}],
     [{ jobs_in_success: 1 }, {}],
     [{ "duplicates.otherOfferId": 1 }, {}],
+    [{ validated: 1 }, {}],
+
+    [{ workplace_siret: 1 }, {}],
+    [{ workplace_brand: 1 }, {}],
+    [{ workplace_legal_name: 1 }, {}],
+    [{ workplace_name: 1 }, {}],
+    [{ workplace_address_label: 1 }, {}],
+    [{ offer_title: 1 }, {}],
+    [{ workplace_naf_label: 1 }, {}],
   ],
-  collectionName,
+  collectionName: "computed_jobs_partners" as const,
 } as const satisfies IModelDescriptor