aucune entite n'est effacée par l'etl, seulement des relations

betagouv · Oct 23, 2024 · 52a8d7d · 52a8d7d
1 parent 673a0f4
commit 52a8d7d
Show file tree

Hide file tree

Showing 9 changed files with 43 additions and 89 deletions.
diff --git a/..._marquage_formations_voeux_deprecated.sql → ...ge_formations_metiers_voeux_obsoletes.sql b/..._marquage_formations_voeux_deprecated.sql → ...ge_formations_metiers_voeux_obsoletes.sql
@@ -1,3 +1,4 @@
 
 ALTER TABLE ref_formation ADD COLUMN obsolete BOOLEAN DEFAULT false NOT NULL;
 ALTER TABLE ref_voeu ADD COLUMN obsolete BOOLEAN DEFAULT false NOT NULL;
+ALTER TABLE ref_metier ADD COLUMN obsolete BOOLEAN DEFAULT false NOT NULL;
diff --git a/app/etl/etl-common/src/main/kotlin/fr/gouv/monprojetsup/data/metier/entity/MetierEntity.kt b/app/etl/etl-common/src/main/kotlin/fr/gouv/monprojetsup/data/metier/entity/MetierEntity.kt
@@ -25,4 +25,8 @@ class MetierEntity {
     @Type(JsonType::class)
     @Column(name = "liens", columnDefinition = "jsonb")
     var liens = arrayListOf<LienEntity>()
+
+    @Column(nullable = false)
+    var obsolete: Boolean = false
+
 }
diff --git a/app/etl/etl-updatedb/src/main/kotlin/fr/gouv/monprojetsup/data/etl/BatchUpdate.kt b/app/etl/etl-updatedb/src/main/kotlin/fr/gouv/monprojetsup/data/etl/BatchUpdate.kt
@@ -10,27 +10,15 @@ class BatchUpdate(
 ) {
 
     fun <T> setEntities(entityName: String, entities: Collection<T> ) {
-
-        clearEntities(entityName)
-        addEntities(entities)
-
-    }
-
-    fun <T> addEntities(entities: Collection<T>) {
         val statelessSession: StatelessSession = sessionFactory.openStatelessSession()
         val transaction: Transaction = statelessSession.beginTransaction()
-        entities.forEach{ statelessSession.insert(it)}
-        transaction.commit()
-        statelessSession.close()
-    }
 
-    fun clearEntities(entityName: String) {
-        val statelessSession: StatelessSession = sessionFactory.openStatelessSession()
-        val transaction: Transaction = statelessSession.beginTransaction()
         val hql = "DELETE FROM $entityName"
         val query = statelessSession.createMutationQuery(hql)
         query.executeUpdate()
 
+        entities.forEach{ statelessSession.insert(it)}
+
         transaction.commit()
         statelessSession.close()
     }
@@ -43,12 +31,4 @@ class BatchUpdate(
         statelessSession.close()
     }
 
-    fun <T> deleteEntities(entities: Collection<T>) {
-        val statelessSession: StatelessSession = sessionFactory.openStatelessSession()
-        val transaction: Transaction = statelessSession.beginTransaction()
-        entities.forEach{ statelessSession.delete(it)}
-        transaction.commit()
-        statelessSession.close()
-    }
-
 }
diff --git a/app/etl/etl-updatedb/src/main/kotlin/fr/gouv/monprojetsup/data/etl/UpdateMpsDb.kt b/app/etl/etl-updatedb/src/main/kotlin/fr/gouv/monprojetsup/data/etl/UpdateMpsDb.kt
@@ -47,13 +47,6 @@ open class Runner(
 		logger.info("Création des fichiers de diagnostic")
 		mpsDataPort.exportDiagnostics()
 
-		//clearAll in this order to avoid foreign key constraint errors
-		logger.info("Vidage des tables métiers, formations, suggestions et référentiels")
-		updateFormationsMetiersDbs.clearAll()
-		updateMetierDbs.clearAll()
-		updateSuggestionsDbs.clearAll()
-		updateReferentielDbs.clearAll()
-
 		logger.info("Mise à jour des référentiel")
 		updateReferentielDbs.updateReferentielDbs()
 

diff --git a/...tl-updatedb/src/main/kotlin/fr/gouv/monprojetsup/data/etl/formation/UpdateFormationDbs.kt b/...tl-updatedb/src/main/kotlin/fr/gouv/monprojetsup/data/etl/formation/UpdateFormationDbs.kt
@@ -107,18 +107,17 @@ class UpdateFormationDbs(
         }
 
         val voeuxIds = voeux.map { it.key }.toSet()
+
         val voeuxObsoletes = HashSet(voeuxDb.findAll())
-        voeuxObsoletes.removeIf { f -> voeuxIds.contains(f.id) }
+        voeuxObsoletes.removeIf { voeuxIds.contains(it.id) }
         if(voeuxObsoletes.isNotEmpty()) {
-            logger.info("Marquage de ${voeuxObsoletes.count()} voeux obsoletes")
+            logger.warning("Marquage de ${voeuxObsoletes.count()} voeux obsoletes")
             voeuxObsoletes.forEach { it.obsolete = true }
             batchUpdate.upsertEntities(voeuxObsoletes)
         }
 
-        batchUpdate.setEntities(
-            VoeuEntity::class.simpleName!!,
-            voeuxEntities
-        )
+        logger.warning("Insertion et mise à jour de ${voeuxEntities.count()} voeux")
+        batchUpdate.upsertEntities(voeuxEntities)
 
     }
 
@@ -145,7 +144,7 @@ class UpdateFormationDbs(
          val formationsObsoletes = HashSet(formationDb.findAll())
          formationsObsoletes.removeIf { f -> formationsMpsIds.contains(f.id) }
          if(formationsObsoletes.isNotEmpty()) {
-             logger.info("Marquage de ${formationsObsoletes.count()} formations obsoletes")
+             logger.warning("Marquage de ${formationsObsoletes.count()} formations obsoletes")
              formationsObsoletes.forEach { it.obsolete = true }
              batchUpdate.upsertEntities(formationsObsoletes)
          }
@@ -211,6 +210,8 @@ class UpdateFormationDbs(
              }
              formationEntities.add(entity)
          }
+
+         logger.warning("Insertion et mise à jour de ${formationEntities.count()} formations")
          batchUpdate.upsertEntities(formationEntities)
      }
 
@@ -261,8 +262,7 @@ class UpdateFormationDbs(
             }
         }
         logger.info("Sauvegarde des correspondances villes-voeux commençant par $letter")
-        batchUpdate.addEntities(entities)
-        entities.clear()
+        batchUpdate.upsertEntities(entities)
     }
 
     /**
@@ -283,7 +283,6 @@ class UpdateFormationDbs(
     }
 
 
-
     private fun updateCriteresDb() {
         val criteres = ArrayList<CritereAnalyseCandidatureEntity>()
         var i = 0
@@ -298,12 +297,5 @@ class UpdateFormationDbs(
         criteresDb.saveAll(criteres)
     }
 
-    fun clearAll() {
-        batchUpdate.clearEntities(MoyenneGeneraleAdmisEntity::class.simpleName!!)
-        batchUpdate.clearEntities(VoeuEntity::class.simpleName!!)
-        batchUpdate.clearEntities(FormationEntity::class.simpleName!!)
-        batchUpdate.clearEntities(CritereAnalyseCandidatureEntity::class.simpleName!!)
-    }
-
 
 }
diff --git a/...src/main/kotlin/fr/gouv/monprojetsup/data/etl/formationmetier/UpdateFormationMetierDbs.kt b/...src/main/kotlin/fr/gouv/monprojetsup/data/etl/formationmetier/UpdateFormationMetierDbs.kt
@@ -47,8 +47,4 @@ class UpdateFormationMetierDbs(
         )
     }
 
-    fun clearAll() {
-        batchUpdate.clearEntities(FormationMetierEntity::class.simpleName!!)
-    }
-
 }
diff --git a/app/etl/etl-updatedb/src/main/kotlin/fr/gouv/monprojetsup/data/etl/metier/UpdateMetierDbs.kt b/app/etl/etl-updatedb/src/main/kotlin/fr/gouv/monprojetsup/data/etl/metier/UpdateMetierDbs.kt
@@ -3,11 +3,11 @@ package fr.gouv.monprojetsup.data.etl.metier
 import fr.gouv.monprojetsup.data.commun.entity.LienEntity
 import fr.gouv.monprojetsup.data.etl.BatchUpdate
 import fr.gouv.monprojetsup.data.etl.MpsDataPort
-import fr.gouv.monprojetsup.data.formationmetier.entity.FormationMetierEntity
 import fr.gouv.monprojetsup.data.metier.entity.MetierEntity
 import org.springframework.data.jpa.repository.JpaRepository
 import org.springframework.stereotype.Component
 import org.springframework.stereotype.Repository
+import java.util.logging.Logger
 
 @Repository
 interface MetiersDb :
@@ -16,11 +16,14 @@ interface MetiersDb :
 @Component
 class UpdateMetierDbs(
     private val mpsDataPort: MpsDataPort,
-    private val batchUpdate: BatchUpdate
+    private val batchUpdate: BatchUpdate,
+    private val metiersDb: MetiersDb
+
 ) {
 
+    private val logger: Logger = Logger.getLogger(UpdateMetierDbs::class.java.simpleName)
+
     fun update() {
-        clearAll()
         updateMetierDb()
     }
 
@@ -32,8 +35,7 @@ class UpdateMetierDbs(
         val metiersAssocies = mpsDataPort.getMetiersAssociesLabels()
 
         val entities =
-            mpsDataPort.getMetiersMpsIds()
-                .map { metierId ->
+            mpsDataPort.getMetiersMpsIds().map { metierId ->
                     val label = labels[metierId]
                     val entity = MetierEntity()
                     if (label != null) {
@@ -54,23 +56,21 @@ class UpdateMetierDbs(
                     }
                     entity
                 }
-        batchUpdate.clearEntities(
-            FormationMetierEntity::class.simpleName!!
-        )
-        batchUpdate.setEntities(
-            MetierEntity::class.simpleName!!,
-            entities
-        )
-    }
 
+        val metiersIds =  entities.map { it.id }.toSet()
+
+        val metiersObsoletes = HashSet(metiersDb.findAll())
+        metiersObsoletes.removeIf { metiersIds.contains(it.id) }
+        if(metiersObsoletes.isNotEmpty()) {
+            logger.warning("Marquage de ${metiersObsoletes.count()} métiers obsoletes")
+            metiersObsoletes.forEach { it.obsolete = true }
+            batchUpdate.upsertEntities(metiersObsoletes)
+        }
+
+        logger.warning("Insertion et mise à jour de ${entities.count()} métiers")
+        batchUpdate.upsertEntities(entities)
 
-    fun clearAll() {
-        batchUpdate.clearEntities(
-            FormationMetierEntity::class.simpleName!!
-        )
-        batchUpdate.clearEntities(
-            MetierEntity::class.simpleName!!
-        )
     }
 
+
 }
diff --git a/...pdatedb/src/main/kotlin/fr/gouv/monprojetsup/data/etl/suggestions/UpdateSuggestionsDbs.kt b/...pdatedb/src/main/kotlin/fr/gouv/monprojetsup/data/etl/suggestions/UpdateSuggestionsDbs.kt
@@ -77,11 +77,7 @@ class UpdateSuggestionsDbs(
             .associateBy { it.id }
             .values
 
-        batchUpdate.setEntities(
-            SuggestionsLabelEntity::class.simpleName!!,
-            entities
-        )
-
+        batchUpdate.upsertEntities(entities)
     }
 
     internal fun updateCandidatsDb() {
@@ -99,10 +95,7 @@ class UpdateSuggestionsDbs(
             .flatMap {  SuggestionsVilleEntity.getEntities(it) }
             .associateBy { it.id }
             .values
-        batchUpdate.setEntities(
-            SuggestionsVilleEntity::class.simpleName!!,
-            entities
-        )
+        batchUpdate.upsertEntities(entities)
     }
 
     internal fun updateEdgesDb() {
@@ -115,13 +108,4 @@ class UpdateSuggestionsDbs(
     }
 
 
-    fun clearAll() {
-        batchUpdate.clearEntities(SuggestionsCandidatEntity::class.simpleName!!)
-        batchUpdate.clearEntities(SuggestionsVilleEntity::class.simpleName!!)
-        batchUpdate.clearEntities(SuggestionsMatiereEntity::class.simpleName!!)
-        batchUpdate.clearEntities(SuggestionsLabelEntity::class.simpleName!!)
-        batchUpdate.clearEntities(SuggestionsEdgeEntity::class.simpleName!!)
-    }
-
-
 }
diff --git a/app/etl/etl-updatedb/src/test/kotlin/fr/gouv/monprojetsup/data/etl/db/UpdateDbsTest.kt b/app/etl/etl-updatedb/src/test/kotlin/fr/gouv/monprojetsup/data/etl/db/UpdateDbsTest.kt
@@ -26,6 +26,7 @@ import org.junit.jupiter.api.assertDoesNotThrow
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.test.context.ActiveProfiles
 import org.springframework.test.context.TestPropertySource
+import kotlin.test.assertNotNull
 
 @TestPropertySource(properties = [
     "logging.level.org.hibernate.SQL=WARN",
@@ -113,9 +114,13 @@ class UpdateDbsTest : BDDRepositoryTest() {
 
             updateFormationDbs.updateFormationsDb()
             updateFormationDbs.updateVoeuxDb()
+
             val formation2 = formationsdb.findById("obsolete").orElse(null)
+            assertNotNull(formation2)
             assertThat(formation2?.obsolete)
+
             val voeu2 = voeuxDb.findById("obsolete").orElse(null)
+            assertNotNull(voeu2)
             assertThat(voeu2?.obsolete)
         }
 
@@ -169,7 +174,6 @@ class UpdateDbsTest : BDDRepositoryTest() {
         lateinit var updateFormationsMetiersDbs: UpdateFormationMetierDbs
 
         private fun update() {
-            updateFormationsMetiersDbs.clearAll()
             updateFormationDbs.updateFormationsDb()
             updateMetierDbs.update()
             updateFormationsMetiersDbs.update()
-Original file line number
+Diff line change
@@ Expand Up / @@ -47,8 +47,4 @@ class UpdateFormationMetierDbs( @@
             )
         }
-        fun clearAll() {
-            batchUpdate.clearEntities(FormationMetierEntity::class.simpleName!!)
-        }
     }