From bbeffb7a74965c516767d1947c519899d7a30ab7 Mon Sep 17 00:00:00 2001 From: Hugo Gimbert Date: Sat, 19 Oct 2024 10:31:41 +0200 Subject: [PATCH] accelere et l + fix test --- .../fr/gouv/monprojetsup/data/TestData.kt | 5 +++ .../entity/SuggestionsVilleEntity.kt | 6 --- .../gouv/monprojetsup/data/etl/BatchUpdate.kt | 12 +++++- .../data/etl/formation/UpdateFormationDbs.kt | 41 +++++++++++++------ .../monprojetsup/data/etl/db/UpdateDbsTest.kt | 7 +--- .../data/etl/port/FormationsMpsTests.kt | 18 ++++---- 6 files changed, 55 insertions(+), 34 deletions(-) diff --git a/app/etl/etl-common/src/main/kotlin/fr/gouv/monprojetsup/data/TestData.kt b/app/etl/etl-common/src/main/kotlin/fr/gouv/monprojetsup/data/TestData.kt index e572c8d2c..daf1aa056 100644 --- a/app/etl/etl-common/src/main/kotlin/fr/gouv/monprojetsup/data/TestData.kt +++ b/app/etl/etl-common/src/main/kotlin/fr/gouv/monprojetsup/data/TestData.kt @@ -17,6 +17,9 @@ class TestData { const val MIN_NB_ARETES_SUGGESTIONS_GRAPH: Long = 1000 const val ECOLE_COMMERCE_PSUP_FR_COD: Int = 24 + const val ECOLE_COMMERCE_PSUP_FL_COD1: Int = 10 * ECOLE_COMMERCE_PSUP_FR_COD + const val ECOLE_COMMERCE_PSUP_FL_COD2: Int = 10 * ECOLE_COMMERCE_PSUP_FR_COD + 1 + const val ECOLE_COMMERCE_PSUP_FL_COD3: Int = 10 * ECOLE_COMMERCE_PSUP_FR_COD + 2 const val CPGE_LETTRES_PSUP_FL_COD: Int = 31 const val CPGE_LETTRE_IDEO_CODE: String = "FOR.1471" const val L1_HISTOIRE_ART_IDEO_CODE: String = "FOR.4741" @@ -40,6 +43,8 @@ class TestData { const val MET_COMMISSAIRE_PRISEUR_IDEO = "MET.673" const val MASTER_TOURISME = "FOR.10361" + const val VILLE_SOULAC_SUR_MER_INSEE_CODE: String = "33514" + val psupToIdeoReference = mapOf( Constants.gFlCodToMpsId(BTS_AERONAUTIQUE_FL_COD_PSUP) to BTS_AERONAUTIQUE_IDEO_COD, Constants.gFlCodToMpsId(CMI_SVT_FL_COD_PSUP) to CMI_SVT_IDEO_COD, diff --git a/app/etl/etl-common/src/main/kotlin/fr/gouv/monprojetsup/data/suggestions/entity/SuggestionsVilleEntity.kt b/app/etl/etl-common/src/main/kotlin/fr/gouv/monprojetsup/data/suggestions/entity/SuggestionsVilleEntity.kt index c0f001310..42e7ce57e 100644 --- a/app/etl/etl-common/src/main/kotlin/fr/gouv/monprojetsup/data/suggestions/entity/SuggestionsVilleEntity.kt +++ b/app/etl/etl-common/src/main/kotlin/fr/gouv/monprojetsup/data/suggestions/entity/SuggestionsVilleEntity.kt @@ -37,12 +37,6 @@ class SuggestionsVilleEntity { nom = it.nom insee = it.codeInsee coords = it.coords - }, - SuggestionsVilleEntity().apply { - id = it.nom - nom = it.nom - insee = it.codeInsee - coords = it.coords } ) } diff --git a/app/etl/etl-updatedb/src/main/kotlin/fr/gouv/monprojetsup/data/etl/BatchUpdate.kt b/app/etl/etl-updatedb/src/main/kotlin/fr/gouv/monprojetsup/data/etl/BatchUpdate.kt index 54c2e6b39..a7bab9bc5 100644 --- a/app/etl/etl-updatedb/src/main/kotlin/fr/gouv/monprojetsup/data/etl/BatchUpdate.kt +++ b/app/etl/etl-updatedb/src/main/kotlin/fr/gouv/monprojetsup/data/etl/BatchUpdate.kt @@ -12,11 +12,11 @@ class BatchUpdate( fun setEntities(entityName: String, entities: Collection ) { clearEntities(entityName) - addEntities(entityName, entities) + addEntities(entities) } - fun addEntities(entityName: String, entities: Collection ) { + fun addEntities(entities: Collection) { val statelessSession: StatelessSession = sessionFactory.openStatelessSession() val transaction: Transaction = statelessSession.beginTransaction() entities.forEach{ statelessSession.insert(it)} @@ -35,4 +35,12 @@ class BatchUpdate( statelessSession.close() } + fun upsertEntities(entities: Collection) { + val statelessSession: StatelessSession = sessionFactory.openStatelessSession() + val transaction: Transaction = statelessSession.beginTransaction() + entities.forEach{ statelessSession.upsert(it)} + transaction.commit() + statelessSession.close() + } + } \ No newline at end of file diff --git a/app/etl/etl-updatedb/src/main/kotlin/fr/gouv/monprojetsup/data/etl/formation/UpdateFormationDbs.kt b/app/etl/etl-updatedb/src/main/kotlin/fr/gouv/monprojetsup/data/etl/formation/UpdateFormationDbs.kt index dc558dfda..1530db020 100644 --- a/app/etl/etl-updatedb/src/main/kotlin/fr/gouv/monprojetsup/data/etl/formation/UpdateFormationDbs.kt +++ b/app/etl/etl-updatedb/src/main/kotlin/fr/gouv/monprojetsup/data/etl/formation/UpdateFormationDbs.kt @@ -46,7 +46,9 @@ class UpdateFormationDbs( private val criteresDb: CriteresDb, private val moyennesGeneralesAdmisDb: MoyennesGeneralesAdmisDb, private val mpsDataPort: MpsDataPort, - private val batchUpdate: BatchUpdate + private val batchUpdate: BatchUpdate, + private val villesVoeuxDb: VillesVoeuxDb + ) { private val logger: Logger = Logger.getLogger(UpdateFormationDbs::class.java.simpleName) @@ -188,10 +190,10 @@ class UpdateFormationDbs( val voeux = mpsDataPort.getVoeux().flatMap { it.value }.toList() - var letter = '_' + logger.info("Récupération des paires villes voeux actuelles") + val villesVoeuxActuels = villesVoeuxDb.findAll().associateBy { v -> v.idVille } - val className = VilleVoeuxEntity::class.simpleName!! - batchUpdate.clearEntities(className) + var letter = '_' val entities = ArrayList() cities.forEach { city -> @@ -199,22 +201,35 @@ class UpdateFormationDbs( if(newLetter != letter) { logger.info("Calcul des distances pour les villes commençant par $newLetter") letter = newLetter - logger.info("Sauvegarde des correspondances villes-voeux commençant par $letter") - batchUpdate.addEntities(className, entities) - entities.clear() + if(entities.isNotEmpty()) { + logger.info("Enregistrement des correspondances villes-voeux commençant par $letter") + batchUpdate.upsertEntities(entities) + entities.clear() + } } - val distances = voeux.map { voeu -> + val currentEntity = villesVoeuxActuels[city.codeInsee] + + val voeuxAlreadyKnow = currentEntity?.distancesVoeuxKm?.keys.orEmpty() + + val nouvellesDistances = voeux + .filter { v -> !voeuxAlreadyKnow.contains(v.id) } + .map { voeu -> voeu.id to geodeticDistance(voeu.coords(), city.coords) } .filter { it.second <= Constants.MAX_DISTANCE_VILLE_VOEU_KM } .toMap() - entities.add(VilleVoeuxEntity().apply { - idVille = city.codeInsee - distancesVoeuxKm = distances - }) + if(nouvellesDistances.isNotEmpty()) { + val distances = HashMap(currentEntity?.distancesVoeuxKm.orEmpty()) + distances.putAll(nouvellesDistances) + val newEntity = VilleVoeuxEntity().apply { + idVille = city.codeInsee + distancesVoeuxKm = distances + } + entities.add(newEntity) + } } logger.info("Sauvegarde des correspondances villes-voeux commençant par $letter") - batchUpdate.addEntities(className, entities) + batchUpdate.addEntities(entities) entities.clear() } diff --git a/app/etl/etl-updatedb/src/test/kotlin/fr/gouv/monprojetsup/data/etl/db/UpdateDbsTest.kt b/app/etl/etl-updatedb/src/test/kotlin/fr/gouv/monprojetsup/data/etl/db/UpdateDbsTest.kt index 659173266..5c5900100 100644 --- a/app/etl/etl-updatedb/src/test/kotlin/fr/gouv/monprojetsup/data/etl/db/UpdateDbsTest.kt +++ b/app/etl/etl-updatedb/src/test/kotlin/fr/gouv/monprojetsup/data/etl/db/UpdateDbsTest.kt @@ -157,13 +157,10 @@ class UpdateDbsTest : BDDRepositoryTest() { } @Test - fun `La table des villes doit inclure Soulac-sur-Mer, à la fois sous son nom et sous son code INSEE`() { + fun `La table des villes doit inclure Soulac-sur-Mer`() { updateSuggestionsDbs.updateVillesDb() - val ville1 = villesDb.findById("33514").map { it.toVille() } - val ville2 = villesDb.findById("Soulac-sur-Mer").map { it.toVille() } + val ville1 = villesDb.findById(TestData.VILLE_SOULAC_SUR_MER_INSEE_CODE) assertThat(ville1).isPresent - assertThat(ville2).isPresent - assertThat(ville1.get()).isEqualTo(ville2.get()) } @Test diff --git a/app/etl/etl-updatedb/src/test/kotlin/fr/gouv/monprojetsup/data/etl/port/FormationsMpsTests.kt b/app/etl/etl-updatedb/src/test/kotlin/fr/gouv/monprojetsup/data/etl/port/FormationsMpsTests.kt index f097285fe..b4e49f01e 100644 --- a/app/etl/etl-updatedb/src/test/kotlin/fr/gouv/monprojetsup/data/etl/port/FormationsMpsTests.kt +++ b/app/etl/etl-updatedb/src/test/kotlin/fr/gouv/monprojetsup/data/etl/port/FormationsMpsTests.kt @@ -1,6 +1,7 @@ package fr.gouv.monprojetsup.data.etl.port -import fr.gouv.monprojetsup.data.Constants +import fr.gouv.monprojetsup.data.Constants.gFlCodToMpsId +import fr.gouv.monprojetsup.data.Constants.gFrCodToMpsId import fr.gouv.monprojetsup.data.TestData import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Nested @@ -79,18 +80,19 @@ class FormationsMpsTests : DataPortTest() { @Test fun `CUPGE - Droit-économie-gestion hérite de tous les métiers des écoles de commerce`() { - //c'est le mapping CPGE - licence qui crée les assciations métiers + //c'est le mapping CPGE - licence qui crée les associations métiers val mpsIds = mpsDataPort.getFormationsMpsIds() - val cupgeEcoGestionMpsId = Constants.gFrCodToMpsId(TestData.CUPGE_ECO_GESTION_PSUP_FR_COD) + val cupgeEcoGestionMpsId = gFrCodToMpsId(TestData.CUPGE_ECO_GESTION_PSUP_FR_COD) assertThat(mpsIds).contains(cupgeEcoGestionMpsId) - val ecoleCommerceMpsId = Constants.gFrCodToMpsId(TestData.ECOLE_COMMERCE_PSUP_FR_COD) - assertThat(mpsIds).contains(ecoleCommerceMpsId) + val ecoleCommerceMpsId = + listOf(TestData.ECOLE_COMMERCE_PSUP_FL_COD1, TestData.ECOLE_COMMERCE_PSUP_FL_COD2,TestData.ECOLE_COMMERCE_PSUP_FL_COD3) + .map { gFlCodToMpsId(it) } + assertThat(mpsIds).containsAll(ecoleCommerceMpsId) val formationsVersMetiers = mpsDataPort.getFormationsVersMetiersEtMetiersAssocies() - val metiersEcoleCommerce = formationsVersMetiers[ecoleCommerceMpsId] - assertThat(metiersEcoleCommerce).isNotNull() + val metiersEcoleCommerce = ecoleCommerceMpsId.flatMap { formationsVersMetiers[it].orEmpty() } assertThat(metiersEcoleCommerce).isNotEmpty() val metiersCupge = formationsVersMetiers[cupgeEcoGestionMpsId] @@ -106,7 +108,7 @@ class FormationsMpsTests : DataPortTest() { fun `CUPGE - Sciences, technologie, santé ne contient pas le mot-clé commerce international `() { val mpsIds = mpsDataPort.getFormationsMpsIds() val cupgeSciencesTechnoSanteMpsId = - Constants.gFrCodToMpsId(TestData.CUPGE_ECO_SCIENCES_TECHNO_SANTE_PSUP_FR_COD) + gFrCodToMpsId(TestData.CUPGE_ECO_SCIENCES_TECHNO_SANTE_PSUP_FR_COD) assertThat(mpsIds).contains(cupgeSciencesTechnoSanteMpsId) assertThat(mpsDataPort.getMotsClesFormations()[cupgeSciencesTechnoSanteMpsId]).doesNotContain("commerce international") }