diff --git a/server/src/jobs/offrePartenaire/detectDuplicateJobPartners.ts b/server/src/jobs/offrePartenaire/detectDuplicateJobPartners.ts index 786892d84..bd8d79c78 100644 --- a/server/src/jobs/offrePartenaire/detectDuplicateJobPartners.ts +++ b/server/src/jobs/offrePartenaire/detectDuplicateJobPartners.ts @@ -3,8 +3,9 @@ import { AnyBulkWriteOperation, Filter } from "mongodb" import { oleoduc, writeData } from "oleoduc" import { RECRUITER_STATUS } from "shared/constants" import { IJob, JOB_STATUS, JOB_STATUS_ENGLISH, ZGlobalAddress } from "shared/models" +import { IComputedJobPartnersDuplicateRef } from "shared/models/jobPartnersDuplicateRef" import jobsPartnersModel, { IJobsPartnersOfferPrivate } from "shared/models/jobsPartners.model" -import jobsPartnersComputedModel, { IComputedJobPartnersDuplicateRef, IComputedJobsPartners, JOB_PARTNER_BUSINESS_ERROR } from "shared/models/jobsPartnersComputed.model" +import jobsPartnersComputedModel, { IComputedJobsPartners, JOB_PARTNER_BUSINESS_ERROR } from "shared/models/jobsPartnersComputed.model" import recruiterModel, { IRecruiter } from "shared/models/recruiter.model" import { removeAccents } from "shared/utils" import * as stringSimilarity from "string-similarity" diff --git a/server/src/jobs/offrePartenaire/fillFieldsForPartnersFactory.ts b/server/src/jobs/offrePartenaire/fillFieldsForPartnersFactory.ts index ff986c4b9..4a316220d 100644 --- a/server/src/jobs/offrePartenaire/fillFieldsForPartnersFactory.ts +++ b/server/src/jobs/offrePartenaire/fillFieldsForPartnersFactory.ts @@ -1,5 +1,5 @@ import { internal } from "@hapi/boom" -import { Filter } from "mongodb" +import { AnyBulkWriteOperation, Filter } from "mongodb" import { oleoduc, writeData } from "oleoduc" import { IJobsPartnersOfferPrivate } from "shared/models/jobsPartners.model" import { COMPUTED_ERROR_SOURCE, IComputedJobsPartners } from "shared/models/jobsPartnersComputed.model" @@ -62,12 +62,14 @@ export const fillFieldsForPartnersFactory = async [field, 1])), _id: 1, + jobs_in_success: 1, }, }) .stream(), @@ -79,16 +81,19 @@ export const fillFieldsForPartnersFactory = async { - const { _id, ...newFields } = document - return [ + const dataToWrite = responses.flatMap((response) => { + const { _id, ...newFields } = response + const updates: AnyBulkWriteOperation[] = [ { updateOne: { filter: { _id }, - update: { $set: newFields }, + update: { $set: { ...newFields, updated_at: now } }, }, }, - { + ] + const document = documents.find((doc2) => doc2._id.equals(_id)) + if (!document?.jobs_in_success.includes(job)) { + updates.push({ updateOne: { filter: { _id }, update: { @@ -97,8 +102,9 @@ export const fillFieldsForPartnersFactory = async { - const filter = { partner_label: { $nin: jobPartnersByFlux } } - + const processId: string = new ObjectId().toString() + await getDbCollection("computed_jobs_partners").updateMany( + { partner_label: { $nin: jobPartnersByFlux }, currently_processed_id: null }, + { $set: { currently_processed_id: processId } } + ) + const filter = { currently_processed_id: processId } await fillComputedJobsPartners(filter) await importFromComputedToJobsPartners(filter) await getDbCollection("computed_jobs_partners").deleteMany({ $and: [filter, { validated: true }] }) + await getDbCollection("computed_jobs_partners").updateMany(filter, { $set: { currently_processed_id: null } }) } diff --git a/server/src/jobs/offrePartenaire/rawToComputedJobsPartners.ts b/server/src/jobs/offrePartenaire/rawToComputedJobsPartners.ts index 4c2cf9fdd..8bfb6e385 100644 --- a/server/src/jobs/offrePartenaire/rawToComputedJobsPartners.ts +++ b/server/src/jobs/offrePartenaire/rawToComputedJobsPartners.ts @@ -46,6 +46,7 @@ export const rawToComputedJobsPartners = async ({ ...computedJobPartner, partner_label: partnerLabel, created_at: importDate, + updated_at: importDate, offer_status_history: [], }) counters.success++ diff --git a/server/src/jobs/offrePartenaire/validateComputedJobPartners.ts b/server/src/jobs/offrePartenaire/validateComputedJobPartners.ts index 2b3864221..a582ac9b3 100644 --- a/server/src/jobs/offrePartenaire/validateComputedJobPartners.ts +++ b/server/src/jobs/offrePartenaire/validateComputedJobPartners.ts @@ -1,7 +1,7 @@ import { AnyBulkWriteOperation, Filter } from "mongodb" import { oleoduc, writeData } from "oleoduc" import jobsPartnersModel from "shared/models/jobsPartners.model" -import { COMPUTED_ERROR_SOURCE, IComputedJobsPartners, JOB_PARTNER_BUSINESS_ERROR } from "shared/models/jobsPartnersComputed.model" +import { COMPUTED_ERROR_SOURCE, IComputedJobsPartners } from "shared/models/jobsPartnersComputed.model" import { logger } from "@/common/logger" import { getDbCollection } from "@/common/utils/mongodbUtils" @@ -55,16 +55,6 @@ export const validateComputedJobPartners = async (addedMatchFilter?: Filter diff --git a/shared/models/jobsPartners.model.ts b/shared/models/jobsPartners.model.ts index 953ed0702..45839066c 100644 --- a/shared/models/jobsPartners.model.ts +++ b/shared/models/jobsPartners.model.ts @@ -6,6 +6,7 @@ import { extensions } from "../helpers/zodHelpers/zodPrimitives.js" import { ZPointGeometry } from "./address.model.js" import { IModelDescriptor, zObjectId } from "./common.js" import { JOB_STATUS_ENGLISH } from "./job.model.js" +import { ZComputedJobPartnersDuplicateRef } from "./jobPartnersDuplicateRef.js" import { zOpcoLabel } from "./opco.model.js" const collectionName = "jobs_partners" as const @@ -111,6 +112,7 @@ export const ZJobsPartnersOfferPrivate = ZJobsPartnersOfferApi.omit({ _id: zObjectId, apply_url: ZJobsPartnersOfferApi.shape.apply_url.nullable().default(null), rank: z.number().nullish().describe("Valeur indiquant la qualité de l'offre. Plus la valeur est élevée, plus la qualité de l'offre est importante"), + duplicates: z.array(ZComputedJobPartnersDuplicateRef).nullish().describe("Référence les autres offres en duplicata avec celle-ci"), }) export const ZJobsPartnersOfferPrivateWithDistance = ZJobsPartnersOfferPrivate.extend({ diff --git a/shared/models/jobsPartnersComputed.model.ts b/shared/models/jobsPartnersComputed.model.ts index d5918bd0e..c2fbf0313 100644 --- a/shared/models/jobsPartnersComputed.model.ts +++ b/shared/models/jobsPartnersComputed.model.ts @@ -4,10 +4,7 @@ import { IModelDescriptor, zObjectId } from "shared/models/common.js" import { extensions } from "../helpers/zodHelpers/zodPrimitives.js" -import jobsPartnersModel, { ZJobsPartnersOfferPrivate } from "./jobsPartners.model.js" -import recruiterModel from "./recruiter.model.js" - -const collectionName = "computed_jobs_partners" as const +import { ZJobsPartnersOfferPrivate } from "./jobsPartners.model.js" export enum COMPUTED_ERROR_SOURCE { API_SIRET = "api_siret", @@ -21,22 +18,11 @@ export enum COMPUTED_ERROR_SOURCE { export enum JOB_PARTNER_BUSINESS_ERROR { CLOSED_COMPANY = "CLOSED_COMPANY", DUPLICATE = "DUPLICATE", - ZOD_VALIDATION = "ZOD_VALIDATION", STAGE = "STAGE", EXPIRED = "EXPIRED", CFA = "CFA", } -export const ZComputedJobPartnersDuplicateRef = z.object({ - otherOfferId: zObjectId, - collectionName: z - .enum([recruiterModel.collectionName, collectionName, jobsPartnersModel.collectionName]) - .describe("nom de la collection contenant l'offre avec _id=otherOfferId"), - reason: z.string(), -}) - -export type IComputedJobPartnersDuplicateRef = z.output - export const ZComputedJobsPartners = extensions .optionalToNullish(ZJobsPartnersOfferPrivate.partial()) .omit({ @@ -60,7 +46,10 @@ export const ZComputedJobsPartners = extensions ), validated: z.boolean().default(false).describe("Toutes les données nécessaires au passage vers jobs_partners sont présentes et valides (validation zod)"), business_error: z.string().nullable().default(null), - duplicates: z.array(ZComputedJobPartnersDuplicateRef).nullish().describe("Référence les autres offres en duplicata avec celle-ci"), + currently_processed_id: z + .string() + .nullish() + .describe("Si le champ est rempli, l'offre est en train d'être traitée. Les offres ayant le même id sont traitées dans le même batch"), }) export type IComputedJobsPartners = z.output @@ -69,12 +58,23 @@ export default { indexes: [ [{ partner_job_id: 1 }, {}], [{ partner_label: 1 }, {}], - [{ validated: 1 }, {}], - [{ errors: 1 }, {}], [{ partner_label: 1, partner_job_id: 1 }, { unique: true }], - [{ workplace_siret: 1 }, {}], + + [{ created_at: 1 }, {}], + [{ updated_at: 1 }, {}], + [{ business_error: 1 }, {}], + [{ errors: 1 }, {}], [{ jobs_in_success: 1 }, {}], [{ "duplicates.otherOfferId": 1 }, {}], + [{ validated: 1 }, {}], + + [{ workplace_siret: 1 }, {}], + [{ workplace_brand: 1 }, {}], + [{ workplace_legal_name: 1 }, {}], + [{ workplace_name: 1 }, {}], + [{ workplace_address_label: 1 }, {}], + [{ offer_title: 1 }, {}], + [{ workplace_naf_label: 1 }, {}], ], - collectionName, + collectionName: "computed_jobs_partners" as const, } as const satisfies IModelDescriptor