From 54b31f1450564008cca124b29ad0f130334aebb2 Mon Sep 17 00:00:00 2001 From: Manuel Soulier Date: Fri, 22 Nov 2024 14:10:00 +0100 Subject: [PATCH] feat: performance problem --- .../java/fr/insee/arc/batch/BatchARC.java | 2 +- .../ApiInitialisationService.java | 2 +- .../ResetEnvironmentService.java | 2 +- .../SynchronizeDataByPilotageOperation.java | 8 ++--- .../dao/SynchronizeDataByPilotageDao.java | 8 +++-- .../ResetEnvironmentOperation.java | 2 +- .../operation/ChargementBrut.java | 32 +------------------ .../main/java/fr/insee/arc/utils/dao/SQL.java | 2 ++ .../fr/insee/arc/utils/utils/FormatSQL.java | 1 + 9 files changed, 17 insertions(+), 42 deletions(-) diff --git a/arc-batch/src/main/java/fr/insee/arc/batch/BatchARC.java b/arc-batch/src/main/java/fr/insee/arc/batch/BatchARC.java index ba27477c7..a185f41a8 100644 --- a/arc-batch/src/main/java/fr/insee/arc/batch/BatchARC.java +++ b/arc-batch/src/main/java/fr/insee/arc/batch/BatchARC.java @@ -775,7 +775,7 @@ private void startMaintenanceThread() { @Override public void run() { try { - DatabaseMaintenance.maintenancePgCatalogAllNods(null, FormatSQL.VACUUM_OPTION_NONE); + DatabaseMaintenance.maintenancePgCatalogAllNods(null, FormatSQL.VACUUM_OPTION_FREEZE); DatabaseMaintenance.maintenancePilotage(null, envExecution, FormatSQL.VACUUM_OPTION_NONE); } catch (ArcException e) { e.logMessageException(); diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/ApiInitialisationService.java b/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/ApiInitialisationService.java index e068a4c33..24bd14158 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/ApiInitialisationService.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/ApiInitialisationService.java @@ -61,7 +61,7 @@ public void executer() throws ArcException { // Met en cohérence les tables de données avec la table de pilotage de // l'environnement // La table de pilotage fait foi - new SynchronizeDataByPilotageOperation(this.coordinatorSandbox).synchronizeDataByPilotage(); + new SynchronizeDataByPilotageOperation(this.coordinatorSandbox).synchronizeDataByPilotage(true); // remettre les archives ou elle doivent etre en cas de restauration de la base new RestoreFileSystem(this.coordinatorSandbox).execute(); diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/ResetEnvironmentService.java b/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/ResetEnvironmentService.java index ac2c17d57..f901b8343 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/ResetEnvironmentService.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/ResetEnvironmentService.java @@ -56,7 +56,7 @@ public static void resetBAS(String env, String rootDirectory) { new ResetEnvironmentOperation(service.getCoordinatorSandbox()).clearPilotageAndDirectories(rootDirectory); // synchronize - new SynchronizeDataByPilotageOperation(service.getCoordinatorSandbox()).synchronizeDataByPilotage(); + new SynchronizeDataByPilotageOperation(service.getCoordinatorSandbox()).synchronizeDataByPilotage(false); } catch (ArcException e) { e.logFullException(); diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/pilotage/SynchronizeDataByPilotageOperation.java b/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/pilotage/SynchronizeDataByPilotageOperation.java index 663ab2bae..2a4fd6f84 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/pilotage/SynchronizeDataByPilotageOperation.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/pilotage/SynchronizeDataByPilotageOperation.java @@ -42,7 +42,7 @@ public SynchronizeDataByPilotageOperation(Sandbox sandbox) { * @param envExecution * @throws ArcException */ - public void synchronizeDataByPilotage() throws ArcException { + public void synchronizeDataByPilotage(boolean fullRebuild) throws ArcException { LoggerHelper.info(LOGGER, "synchronisationEnvironmentByPilotage"); // maintenance de la table de pilotage @@ -54,7 +54,7 @@ public void synchronizeDataByPilotage() throws ArcException { resetEtapePilotage(); // recrée la table de pilotage, ses index, son trigger - rebuildPilotage(); + rebuildPilotage(fullRebuild); // drop des tables temporaires de travail dropUnusedTemporaryTablesAllNods(); @@ -88,8 +88,8 @@ private void resetEtapePilotage() throws ArcException { - private void rebuildPilotage() throws ArcException { - SynchronizeDataByPilotageDao.rebuildPilotageDao(this.sandbox.getConnection(), this.sandbox.getSchema()); + private void rebuildPilotage(boolean fullRebuild) throws ArcException { + SynchronizeDataByPilotageDao.rebuildPilotageDao(this.sandbox.getConnection(), this.sandbox.getSchema(), fullRebuild); } diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/pilotage/dao/SynchronizeDataByPilotageDao.java b/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/pilotage/dao/SynchronizeDataByPilotageDao.java index 896f3ff79..92f5ed770 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/pilotage/dao/SynchronizeDataByPilotageDao.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/pilotage/dao/SynchronizeDataByPilotageDao.java @@ -71,11 +71,13 @@ public static void resetEtapePilotageDao(Connection connection, String envExecut * @param envExecution * @throws ArcException */ - public static void rebuildPilotageDao(Connection connexion, String envExecution) throws ArcException { + public static void rebuildPilotageDao(Connection connexion, String envExecution, boolean fullRebuild) throws ArcException { String tablePilotage = ViewEnum.PILOTAGE_FICHIER.getFullName(envExecution); - - UtilitaireDao.get(0).executeRequest(connexion, new ArcPreparedStatementBuilder("analyze " + tablePilotage + ";")); + + UtilitaireDao.get(0).executeImmediate(connexion, FormatSQL.vacuumSecured(tablePilotage, fullRebuild?FormatSQL.VACUUM_OPTION_FULL:FormatSQL.VACUUM_OPTION_NONE)); + + UtilitaireDao.get(0).executeImmediate(connexion, FormatSQL.analyzeSecured(tablePilotage)); } /** diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/useroperation/ResetEnvironmentOperation.java b/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/useroperation/ResetEnvironmentOperation.java index d97a16440..c8574b914 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/useroperation/ResetEnvironmentOperation.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/useroperation/ResetEnvironmentOperation.java @@ -75,7 +75,7 @@ public void retourPhasePrecedente(TraitementPhase phase, List querySelec nbLignes += resetEnvironmentDao.executeDeletePhaseEntriesInPilotageAndCount(phase, querySelection); // Run a database synchronization with the pilotage table - new SynchronizeDataByPilotageOperation(this.sandbox).synchronizeDataByPilotage(); + new SynchronizeDataByPilotageOperation(this.sandbox).synchronizeDataByPilotage(false); if (nbLignes > 0) { DatabaseMaintenance.maintenanceDatabaseClassic(connection, envExecution); diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/operation/ChargementBrut.java b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/operation/ChargementBrut.java index 4ac9c8371..c087d95f3 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/operation/ChargementBrut.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/operation/ChargementBrut.java @@ -119,11 +119,7 @@ public void calculeNormeAndValiditeFichiers(InputStream file, FileIdCard normeOk if (normeOk.getIdNorme()==null) { throw new ArcException(ArcExceptionMessage.LOAD_NORM_NOT_FOUND, normeOk.getIdSource()); } - - if (estHorsCalendrier(normeOk, envExecution)) { - throw new ArcException(ArcExceptionMessage.LOAD_NORM_OUT_OF_CALENDAR, normeOk.getIdNorme(), normeOk.getIdSource()); - } - + } /** Retourne (par référence) la norme dans normeOk[0] et la validité dans validiteOk[0]. @@ -177,32 +173,6 @@ private void calculerNormeAndValidite(FileIdCard normeOk, String requeteFichier) } - /** Retourne VRAI si la date de validité du fichier est hors de la période de validité de la norme du fichier. - * Retourne FAUX sinon. - * @param normeOk les métadonnées du fichier à vérifier - * @param envExecution le schéma d'exécution du traitement - * @throws ArcException si aucune norme ou plus d'une norme trouvée - */ - private boolean estHorsCalendrier(FileIdCard normeOk, String envExecution) throws ArcException { - ArcPreparedStatementBuilder query = new ArcPreparedStatementBuilder(); - query.build(SQL.SELECT, ColumnEnum.VALIDITE_INF.getColumnName(), ","); - query.build(ColumnEnum.VALIDITE_SUP.getColumnName(), SQL.FROM, ViewEnum.CALENDRIER.getFullName(envExecution)); - query.build(SQL.WHERE, ColumnEnum.ID_NORME.getColumnName(), "=", query.quoteText(normeOk.getIdNorme())); - - List> result = UtilitaireDao.get(0).executeRequestWithoutMetadata(this.connexion, query); - - if (result.size() > 1) { - throw new ArcException(ArcExceptionMessage.LOAD_SEVERAL_NORM_FOUND, normeOk.getIdNorme()); - } else if (result.isEmpty()) { - throw new ArcException(ArcExceptionMessage.LOAD_ZERO_NORM_FOUND); - } - - String validite_inf = result.get(0).get(0); - String validite_sup = result.get(0).get(1); - String validite = normeOk.getValidite(); - return (validite.compareTo(validite_inf) < 0) || (validite.compareTo(validite_sup) > 0); - - } /** * @return the connexion diff --git a/arc-utils/src/main/java/fr/insee/arc/utils/dao/SQL.java b/arc-utils/src/main/java/fr/insee/arc/utils/dao/SQL.java index 4021f3fb8..bf4a1271c 100644 --- a/arc-utils/src/main/java/fr/insee/arc/utils/dao/SQL.java +++ b/arc-utils/src/main/java/fr/insee/arc/utils/dao/SQL.java @@ -3,6 +3,8 @@ public enum SQL { SELECT("SELECT"), FROM("FROM"), WHERE("WHERE"), ORDER_BY("ORDER BY"), DISTINCT("DISTINCT"), + DESC("desc"), ASC("asc"), + AS("AS"), VACUUM("VACUUM"), diff --git a/arc-utils/src/main/java/fr/insee/arc/utils/utils/FormatSQL.java b/arc-utils/src/main/java/fr/insee/arc/utils/utils/FormatSQL.java index eb3235f79..ae56815f9 100644 --- a/arc-utils/src/main/java/fr/insee/arc/utils/utils/FormatSQL.java +++ b/arc-utils/src/main/java/fr/insee/arc/utils/utils/FormatSQL.java @@ -39,6 +39,7 @@ private FormatSQL() { public static final int TIMEOUT_MAINTENANCE = 600000; public static final String VACUUM_OPTION_NONE = ""; + public static final String VACUUM_OPTION_FREEZE = "freeze"; public static final String VACUUM_OPTION_FULL = "full"; public static final String VACUUM_OPTION_ANALYZE = "analyze";