accelere et l + fix test

app/etl/etl-common/src/main/kotlin/fr/gouv/monprojetsup/data/TestData.kt

@@ -17,6 +17,9 @@ class TestData {
         const val MIN_NB_ARETES_SUGGESTIONS_GRAPH: Long = 1000
 
         const val ECOLE_COMMERCE_PSUP_FR_COD: Int = 24
+        const val ECOLE_COMMERCE_PSUP_FL_COD1: Int = 10 * ECOLE_COMMERCE_PSUP_FR_COD
+        const val ECOLE_COMMERCE_PSUP_FL_COD2: Int = 10 * ECOLE_COMMERCE_PSUP_FR_COD + 1
+        const val ECOLE_COMMERCE_PSUP_FL_COD3: Int = 10 * ECOLE_COMMERCE_PSUP_FR_COD + 2
         const val CPGE_LETTRES_PSUP_FL_COD: Int = 31
         const val CPGE_LETTRE_IDEO_CODE: String = "FOR.1471"
         const val L1_HISTOIRE_ART_IDEO_CODE: String = "FOR.4741"
@@ -40,6 +43,8 @@ class TestData {
         const val MET_COMMISSAIRE_PRISEUR_IDEO = "MET.673"
         const val MASTER_TOURISME = "FOR.10361"
 
+        const val VILLE_SOULAC_SUR_MER_INSEE_CODE: String = "33514"
+
         val psupToIdeoReference = mapOf(
             Constants.gFlCodToMpsId(BTS_AERONAUTIQUE_FL_COD_PSUP) to BTS_AERONAUTIQUE_IDEO_COD,
             Constants.gFlCodToMpsId(CMI_SVT_FL_COD_PSUP) to CMI_SVT_IDEO_COD,

app/etl/etl-common/src/main/kotlin/fr/gouv/monprojetsup/data/suggestions/entity/SuggestionsVilleEntity.kt

@@ -37,12 +37,6 @@ class SuggestionsVilleEntity {
                     nom = it.nom
                     insee = it.codeInsee
                     coords = it.coords
-                },
-                SuggestionsVilleEntity().apply {
-                    id = it.nom
-                    nom = it.nom
-                    insee = it.codeInsee
-                    coords = it.coords
                 }
             )
         }

app/etl/etl-updatedb/src/main/kotlin/fr/gouv/monprojetsup/data/etl/BatchUpdate.kt

@@ -12,11 +12,11 @@ class BatchUpdate(
     fun <T> setEntities(entityName: String, entities: Collection<T> ) {
 
         clearEntities(entityName)
-        addEntities(entityName, entities)
+        addEntities(entities)
 
     }
 
-    fun <T> addEntities(entityName: String, entities: Collection<T> ) {
+    fun <T> addEntities(entities: Collection<T>) {
         val statelessSession: StatelessSession = sessionFactory.openStatelessSession()
         val transaction: Transaction = statelessSession.beginTransaction()
         entities.forEach{ statelessSession.insert(it)}
@@ -35,4 +35,12 @@ class BatchUpdate(
         statelessSession.close()
     }
 
+    fun <T> upsertEntities(entities: Collection<T>) {
+        val statelessSession: StatelessSession = sessionFactory.openStatelessSession()
+        val transaction: Transaction = statelessSession.beginTransaction()
+        entities.forEach{ statelessSession.upsert(it)}
+        transaction.commit()
+        statelessSession.close()
+    }
+
 }

app/etl/etl-updatedb/src/main/kotlin/fr/gouv/monprojetsup/data/etl/formation/UpdateFormationDbs.kt

@@ -46,7 +46,9 @@ class UpdateFormationDbs(
     private val criteresDb: CriteresDb,
     private val moyennesGeneralesAdmisDb: MoyennesGeneralesAdmisDb,
     private val mpsDataPort: MpsDataPort,
-    private val batchUpdate: BatchUpdate
+    private val batchUpdate: BatchUpdate,
+    private val villesVoeuxDb: VillesVoeuxDb
+
 ) {
 
     private val logger: Logger = Logger.getLogger(UpdateFormationDbs::class.java.simpleName)
@@ -188,33 +190,46 @@ class UpdateFormationDbs(
 
         val voeux = mpsDataPort.getVoeux().flatMap { it.value }.toList()
 
-        var letter = '_'
+        logger.info("Récupération des paires villes voeux actuelles")
+        val villesVoeuxActuels = villesVoeuxDb.findAll().associateBy { v -> v.idVille }
 
-        val className = VilleVoeuxEntity::class.simpleName!!
-        batchUpdate.clearEntities(className)
+        var letter = '_'
 
         val entities = ArrayList<VilleVoeuxEntity>()
         cities.forEach { city ->
             val newLetter = city.nom.first()
             if(newLetter != letter) {
                 logger.info("Calcul des distances pour les villes commençant par $newLetter")
                 letter = newLetter
-                logger.info("Sauvegarde des correspondances villes-voeux commençant par $letter")
-                batchUpdate.addEntities(className, entities)
-                entities.clear()
+                if(entities.isNotEmpty()) {
+                    logger.info("Enregistrement des correspondances villes-voeux commençant par $letter")
+                    batchUpdate.upsertEntities(entities)
+                    entities.clear()
+                }
             }
-            val distances = voeux.map { voeu ->
+            val currentEntity = villesVoeuxActuels[city.codeInsee]
+
+            val voeuxAlreadyKnow = currentEntity?.distancesVoeuxKm?.keys.orEmpty()
+
+            val nouvellesDistances = voeux
+                .filter { v -> !voeuxAlreadyKnow.contains(v.id) }
+                .map { voeu ->
                 voeu.id to geodeticDistance(voeu.coords(), city.coords)
             }
                 .filter { it.second <= Constants.MAX_DISTANCE_VILLE_VOEU_KM }
                 .toMap()
-            entities.add(VilleVoeuxEntity().apply {
-                idVille = city.codeInsee
-                distancesVoeuxKm = distances
-            })
+            if(nouvellesDistances.isNotEmpty()) {
+                val distances = HashMap(currentEntity?.distancesVoeuxKm.orEmpty())
+                distances.putAll(nouvellesDistances)
+                val newEntity = VilleVoeuxEntity().apply {
+                    idVille = city.codeInsee
+                    distancesVoeuxKm = distances
+                }
+                entities.add(newEntity)
+            }
         }
         logger.info("Sauvegarde des correspondances villes-voeux commençant par $letter")
-        batchUpdate.addEntities(className, entities)
+        batchUpdate.addEntities(entities)
         entities.clear()
     }
 

app/etl/etl-updatedb/src/test/kotlin/fr/gouv/monprojetsup/data/etl/db/UpdateDbsTest.kt

@@ -157,13 +157,10 @@ class UpdateDbsTest : BDDRepositoryTest() {
         }
 
         @Test
-        fun `La table des villes doit inclure Soulac-sur-Mer, à la fois sous son nom et sous son code INSEE`() {
+        fun `La table des villes doit inclure Soulac-sur-Mer`() {
             updateSuggestionsDbs.updateVillesDb()
-            val ville1 = villesDb.findById("33514").map { it.toVille() }
-            val ville2 = villesDb.findById("Soulac-sur-Mer").map { it.toVille() }
+            val ville1 = villesDb.findById(TestData.VILLE_SOULAC_SUR_MER_INSEE_CODE)
             assertThat(ville1).isPresent
-            assertThat(ville2).isPresent
-            assertThat(ville1.get()).isEqualTo(ville2.get())
         }
 
         @Test

app/etl/etl-updatedb/src/test/kotlin/fr/gouv/monprojetsup/data/etl/port/FormationsMpsTests.kt

@@ -1,6 +1,7 @@
 package fr.gouv.monprojetsup.data.etl.port
 
-import fr.gouv.monprojetsup.data.Constants
+import fr.gouv.monprojetsup.data.Constants.gFlCodToMpsId
+import fr.gouv.monprojetsup.data.Constants.gFrCodToMpsId
 import fr.gouv.monprojetsup.data.TestData
 import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.Nested
@@ -79,18 +80,19 @@ class FormationsMpsTests : DataPortTest() {
 
         @Test
         fun `CUPGE - Droit-économie-gestion hérite de tous les métiers des écoles de commerce`() {
-            //c'est le mapping CPGE - licence qui crée les assciations métiers
+            //c'est le mapping CPGE - licence qui crée les associations métiers
             val mpsIds = mpsDataPort.getFormationsMpsIds()
-            val cupgeEcoGestionMpsId = Constants.gFrCodToMpsId(TestData.CUPGE_ECO_GESTION_PSUP_FR_COD)
+            val cupgeEcoGestionMpsId = gFrCodToMpsId(TestData.CUPGE_ECO_GESTION_PSUP_FR_COD)
             assertThat(mpsIds).contains(cupgeEcoGestionMpsId)
 
-            val ecoleCommerceMpsId = Constants.gFrCodToMpsId(TestData.ECOLE_COMMERCE_PSUP_FR_COD)
-            assertThat(mpsIds).contains(ecoleCommerceMpsId)
+            val ecoleCommerceMpsId =
+                listOf(TestData.ECOLE_COMMERCE_PSUP_FL_COD1, TestData.ECOLE_COMMERCE_PSUP_FL_COD2,TestData.ECOLE_COMMERCE_PSUP_FL_COD3)
+                    .map { gFlCodToMpsId(it) }
+            assertThat(mpsIds).containsAll(ecoleCommerceMpsId)
 
             val formationsVersMetiers = mpsDataPort.getFormationsVersMetiersEtMetiersAssocies()
 
-            val metiersEcoleCommerce = formationsVersMetiers[ecoleCommerceMpsId]
-            assertThat(metiersEcoleCommerce).isNotNull()
+            val metiersEcoleCommerce = ecoleCommerceMpsId.flatMap { formationsVersMetiers[it].orEmpty() }
             assertThat(metiersEcoleCommerce).isNotEmpty()
 
             val metiersCupge = formationsVersMetiers[cupgeEcoGestionMpsId]
@@ -106,7 +108,7 @@ class FormationsMpsTests : DataPortTest() {
     fun `CUPGE - Sciences, technologie, santé ne contient pas le mot-clé commerce international `() {
         val mpsIds = mpsDataPort.getFormationsMpsIds()
         val cupgeSciencesTechnoSanteMpsId =
-            Constants.gFrCodToMpsId(TestData.CUPGE_ECO_SCIENCES_TECHNO_SANTE_PSUP_FR_COD)
+            gFrCodToMpsId(TestData.CUPGE_ECO_SCIENCES_TECHNO_SANTE_PSUP_FR_COD)
         assertThat(mpsIds).contains(cupgeSciencesTechnoSanteMpsId)
         assertThat(mpsDataPort.getMotsClesFormations()[cupgeSciencesTechnoSanteMpsId]).doesNotContain("commerce international")
     }