From 0cf606a022a2f0773705a397a2d44b97f01c81e2 Mon Sep 17 00:00:00 2001 From: Manuel Soulier Date: Sun, 8 Oct 2023 17:03:43 +0200 Subject: [PATCH] feat: fileIdCard --- .../arc/core/dataobjects/ColumnEnum.java | 2 + .../core/service/global/bo/NormeFichier.java | 38 ------ .../service/global/dao/RulesOperations.java | 16 +-- .../p2chargement/ApiChargementService.java | 8 +- .../p2chargement/bo/FileAttributes.java | 82 ------------ .../p2chargement/bo/FileAttributesCSV.java | 30 +++++ .../service/p2chargement/bo/FileIdCard.java | 83 ++++++++++++ ...rgement.java => FileIdCardChargement.java} | 4 +- .../bo/{Norme.java => NormeRules.java} | 22 ++- .../p2chargement/dao/ChargeurCsvDao.java | 29 ++-- .../p2chargement/engine/ChargementBrut.java | 37 +++--- .../p2chargement/engine/ChargeurCSV.java | 36 +++-- .../engine/ChargeurClefValeur.java | 39 +++--- .../p2chargement/engine/ChargeurXml.java | 19 ++- .../engine/ChargeurXmlComplexe.java | 49 +++---- .../engine/ParseFormatRulesOperation.java | 10 +- .../p2chargement/factory/ChargeurFactory.java | 10 +- .../thread/ThreadChargementService.java | 87 ++++++------ .../p2chargement/xmlhandler/ArbreFormat.java | 7 +- .../xmlhandler/XMLComplexeHandlerCharger.java | 31 ++--- .../xmlhandler/XMLHandlerCharger4.java | 22 ++- .../xmlhandler/XMLHandlerUtility.java | 125 ++++++++++++++++++ .../utils/exception/ArcExceptionMessage.java | 1 + .../execute/ExecuteEngineController.java | 10 +- 24 files changed, 449 insertions(+), 348 deletions(-) delete mode 100644 arc-core/src/main/java/fr/insee/arc/core/service/global/bo/NormeFichier.java delete mode 100644 arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/FileAttributes.java create mode 100644 arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/FileAttributesCSV.java create mode 100644 arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/FileIdCard.java rename arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/{RegleChargement.java => FileIdCardChargement.java} (83%) rename arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/{Norme.java => NormeRules.java} (76%) create mode 100644 arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/xmlhandler/XMLHandlerUtility.java diff --git a/arc-core/src/main/java/fr/insee/arc/core/dataobjects/ColumnEnum.java b/arc-core/src/main/java/fr/insee/arc/core/dataobjects/ColumnEnum.java index 8dad30f2b..dea73ef5b 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/dataobjects/ColumnEnum.java +++ b/arc-core/src/main/java/fr/insee/arc/core/dataobjects/ColumnEnum.java @@ -19,6 +19,7 @@ public enum ColumnEnum { , CONDITION("condition", TypeEnum.TEXT, "") // ihm_controle_regle , DATE_INACTIF("date_inactif", TypeEnum.DATE, "") // ihm_jeuderegle , DATE_PRODUCTION("date_production", TypeEnum.DATE, "") // ihm_jeuderegle + , DATE_INTEGRATION("date_integration", TypeEnum.TEXT, "") , DEF_NORME("def_norme", TypeEnum.TEXT, "") // ihm_norme , DEF_VALIDITE("def_validite", TypeEnum.TEXT, "") // ihm_norme , DELIMITER("delimiter", TypeEnum.TEXT, "") // ihm_chargement_regle @@ -38,6 +39,7 @@ public enum ColumnEnum { , FORMAT("format", TypeEnum.TEXT, "") // ihm_chargement_regle , HOST_ALLOWED("host_allowed", TypeEnum.TEXT, "") // ihm_webservice_whitelist,ihm_webservice_log , ID("id", TypeEnum.TEXT, "") // ext_webservice_type,ext_mod_periodicite,ext_webservice_queryview,ext_export_format,ext_etat,ext_type_normage,ext_type_fichier_chargement,ext_type_controle,ext_etat_jeuderegle + , ID_SAX("id", TypeEnum.INTEGER, "") // used by sax xml loaders , IDEP("idep", TypeEnum.TEXT, "") // ihm_user , ID_APPLICATION("id_application", TypeEnum.TEXT, "") // ihm_webservice_whitelist,ihm_webservice_log,ihm_client , ID_CLASSE("id_classe", TypeEnum.TEXT, "") // ihm_normage_regle,ihm_controle_regle diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/global/bo/NormeFichier.java b/arc-core/src/main/java/fr/insee/arc/core/service/global/bo/NormeFichier.java deleted file mode 100644 index cf11b1fc2..000000000 --- a/arc-core/src/main/java/fr/insee/arc/core/service/global/bo/NormeFichier.java +++ /dev/null @@ -1,38 +0,0 @@ -package fr.insee.arc.core.service.global.bo; - -public class NormeFichier { - private String idNorme; - private String validite; - private String periodicite; - - - - public NormeFichier(String idNorme, String validite, String periodicite) { - super(); - this.idNorme = idNorme; - this.validite = validite; - this.periodicite = periodicite; - } - - public String getIdNorme() { - return idNorme; - } - public void setIdNorme(String idNorme) { - this.idNorme = idNorme; - } - public String getValidite() { - return validite; - } - public void setValidite(String validite) { - this.validite = validite; - } - public String getPeriodicite() { - return periodicite; - } - public void setPeriodicite(String periodicite) { - this.periodicite = periodicite; - } - - -} - diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/global/dao/RulesOperations.java b/arc-core/src/main/java/fr/insee/arc/core/service/global/dao/RulesOperations.java index 2b4476ef8..720e9a7f9 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/global/dao/RulesOperations.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/global/dao/RulesOperations.java @@ -7,7 +7,7 @@ import fr.insee.arc.core.dataobjects.ArcPreparedStatementBuilder; import fr.insee.arc.core.dataobjects.ColumnEnum; import fr.insee.arc.core.service.global.bo.ArcDateFormat; -import fr.insee.arc.core.service.global.bo.NormeFichier; +import fr.insee.arc.core.service.p2chargement.bo.FileIdCard; import fr.insee.arc.utils.dao.UtilitaireDao; import fr.insee.arc.utils.exception.ArcException; import fr.insee.arc.utils.structure.GenericBean; @@ -34,10 +34,10 @@ public static String getRegles(String tableRegle, String tablePilotage) { return requete.toString(); } - public static String getRegles(String tableRegle, NormeFichier normeFichier) { + public static String getRegles(String tableRegle, FileIdCard fileIdCard) { StringBuilder requete = new StringBuilder(); requete.append("\n SELECT * FROM " + tableRegle + " a WHERE "); - requete.append(conditionRegle(normeFichier)); + requete.append(conditionRegle(fileIdCard)); return requete.toString(); } @@ -109,13 +109,13 @@ private static String conditionRegle(String tablePilotage) { return requete.toString(); } - private static String conditionRegle(NormeFichier normeFichier) { + private static String conditionRegle(FileIdCard fileIdCard) { StringBuilder requete = new StringBuilder(); requete.append("\n "); - requete.append("a.id_norme='" + normeFichier.getIdNorme() + "' "); - requete.append("AND a.periodicite='" + normeFichier.getPeriodicite() + "' "); - requete.append("AND a.validite_inf<=to_date('" + normeFichier.getValidite() + "','"+ArcDateFormat.DATE_FORMAT_CONVERSION.getDatastoreFormat()+"') "); - requete.append("AND a.validite_sup>=to_date('" + normeFichier.getValidite() + "','"+ArcDateFormat.DATE_FORMAT_CONVERSION.getDatastoreFormat()+"') "); + requete.append("a.id_norme='" + fileIdCard.getIdNorme() + "' "); + requete.append("AND a.periodicite='" + fileIdCard.getPeriodicite() + "' "); + requete.append("AND a.validite_inf<=to_date('" + fileIdCard.getValidite() + "','"+ArcDateFormat.DATE_FORMAT_CONVERSION.getDatastoreFormat()+"') "); + requete.append("AND a.validite_sup>=to_date('" + fileIdCard.getValidite() + "','"+ArcDateFormat.DATE_FORMAT_CONVERSION.getDatastoreFormat()+"') "); requete.append(";"); return requete.toString(); } diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/ApiChargementService.java b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/ApiChargementService.java index 30c58160d..5307dea0e 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/ApiChargementService.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/ApiChargementService.java @@ -13,7 +13,7 @@ import fr.insee.arc.core.service.global.ApiService; import fr.insee.arc.core.service.global.thread.MultiThreading; import fr.insee.arc.core.service.p1reception.provider.DirectoriesReception; -import fr.insee.arc.core.service.p2chargement.bo.Norme; +import fr.insee.arc.core.service.p2chargement.bo.NormeRules; import fr.insee.arc.core.service.p2chargement.thread.ThreadChargementService; import fr.insee.arc.core.util.BDParameters; import fr.insee.arc.core.util.StaticLoggerDispatcher; @@ -50,7 +50,7 @@ public ApiChargementService(String aCurrentPhase, String aEnvExecution, String a super(aCurrentPhase, aEnvExecution, aDirectoryRoot, aNbEnr, paramBatch); } - protected List listeNorme; + protected List listeNorme; protected String directoryIn; @Override @@ -65,7 +65,7 @@ public void executer() throws ArcException { this.directoryIn = new DirectoriesReception(this.coordinatorSandbox).getDirectoryReceptionOK() + File.separator; // récupération des différentes normes dans la base - this.listeNorme = Norme.getNormesBase(this.connexion.getCoordinatorConnection(), this.envExecution); + this.listeNorme = NormeRules.getNormesBase(this.connexion.getCoordinatorConnection(), this.envExecution); this.maxParallelWorkers = bdParameters.getInt(this.connexion.getCoordinatorConnection(), "ApiChargementService.MAX_PARALLEL_WORKERS", 4); @@ -82,7 +82,7 @@ public void executer() throws ArcException { } - public List getListeNorme() { + public List getListeNorme() { return listeNorme; } diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/FileAttributes.java b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/FileAttributes.java deleted file mode 100644 index e1596e36e..000000000 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/FileAttributes.java +++ /dev/null @@ -1,82 +0,0 @@ -package fr.insee.arc.core.service.p2chargement.bo; - -import java.util.Date; - -import fr.insee.arc.core.service.global.dao.DateConversion; - -public class FileAttributes { - - - private String[] headers; - private String[] headersV; - private String[] headersI; - private String fileName; - private String validite; - private final String integrationDate = DateConversion.queryDateConversion(new Date()); - - - public FileAttributes(String fileName, String validite) { - super(); - this.fileName = fileName; - this.validite = validite; - } - - - public String[] getHeaders() { - return headers; - } - - - public void setHeaders(String[] headers) { - this.headers = headers; - } - - - public String[] getHeadersV() { - return headersV; - } - - - public void setHeadersV(String[] headersV) { - this.headersV = headersV; - } - - - public String[] getHeadersI() { - return headersI; - } - - - public void setHeadersI(String[] headersI) { - this.headersI = headersI; - } - - - public String getFileName() { - return fileName; - } - - - public void setFileName(String fileName) { - this.fileName = fileName; - } - - - public String getValidite() { - return validite; - } - - - public void setValidite(String validite) { - this.validite = validite; - } - - - public String getIntegrationDate() { - return integrationDate; - } - - - - -} diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/FileAttributesCSV.java b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/FileAttributesCSV.java new file mode 100644 index 000000000..baf77f55f --- /dev/null +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/FileAttributesCSV.java @@ -0,0 +1,30 @@ +package fr.insee.arc.core.service.p2chargement.bo; + +public class FileAttributesCSV { + + + private String[] headersV; + private String[] headersI; + + + public String[] getHeadersV() { + return headersV; + } + + + public void setHeadersV(String[] headersV) { + this.headersV = headersV; + } + + + public String[] getHeadersI() { + return headersI; + } + + + public void setHeadersI(String[] headersI) { + this.headersI = headersI; + } + + +} diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/FileIdCard.java b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/FileIdCard.java new file mode 100644 index 000000000..0e55afc9a --- /dev/null +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/FileIdCard.java @@ -0,0 +1,83 @@ +package fr.insee.arc.core.service.p2chargement.bo; + +import java.util.Date; + +import fr.insee.arc.core.service.global.dao.DateConversion; + +public class FileIdCard { + + + private String fileName; + private String idNorme; + private String validite; + private String periodicite; + private final String integrationDate = DateConversion.queryDateConversion(new Date()); + + private FileIdCardChargement regleChargement; + public FileIdCardChargement getRegleChargement() { + return regleChargement; + } + public void setRegleChargement(FileIdCardChargement regleChargement) { + this.regleChargement = regleChargement; + } + + + public FileIdCard(String idSource) { + this.fileName = idSource; + } + + public void setFileAttributes(String idNorme, String validite, String periodicite) { + this.idNorme = idNorme; + this.validite = validite; + this.periodicite = periodicite; + } + + + public String getFileName() { + return fileName; + } + + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + + public String getValidite() { + return validite; + } + + + public void setValidite(String validite) { + this.validite = validite; + } + + + public String getIntegrationDate() { + return integrationDate; + } + + + public String getIdNorme() { + return idNorme; + } + + + public void setIdNorme(String idNorme) { + this.idNorme = idNorme; + } + + + public String getPeriodicite() { + return periodicite; + } + + + public void setPeriodicite(String periodicite) { + this.periodicite = periodicite; + } + + + + +} diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/RegleChargement.java b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/FileIdCardChargement.java similarity index 83% rename from arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/RegleChargement.java rename to arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/FileIdCardChargement.java index 2d7b8fddf..e5fc49910 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/RegleChargement.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/FileIdCardChargement.java @@ -7,14 +7,14 @@ * @author S4LWO8 * */ -public class RegleChargement { +public class FileIdCardChargement { private TypeChargement typeChargement; private String format; private String delimiter; - public RegleChargement(TypeChargement typeChargement, String delimiter, String format) { + public FileIdCardChargement(TypeChargement typeChargement, String delimiter, String format) { super(); this.typeChargement = typeChargement; this.format = format; diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/Norme.java b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/NormeRules.java similarity index 76% rename from arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/Norme.java rename to arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/NormeRules.java index 0a3f6dbb1..817a52d1e 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/Norme.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/bo/NormeRules.java @@ -21,17 +21,16 @@ * @author S4LWO8 * */ -public class Norme { +public class NormeRules { private String idNorme; private String periodicite; private String defNorme; private String defValidite; - private RegleChargement regleChargement; - private static final Logger LOGGER = LogManager.getLogger(Norme.class); + private static final Logger LOGGER = LogManager.getLogger(NormeRules.class); - public Norme(String idNorme, String periodicite, String defNorme, String defValidite) { + public NormeRules(String idNorme, String periodicite, String defNorme, String defValidite) { super(); this.idNorme = idNorme; this.periodicite = periodicite; @@ -39,7 +38,7 @@ public Norme(String idNorme, String periodicite, String defNorme, String defVali this.defValidite = defValidite; } - public Norme() { + public NormeRules() { } public String getIdNorme() { @@ -66,12 +65,7 @@ public String getDefValidite() { public void setDefValidite(String defValidite) { this.defValidite = defValidite; } - public RegleChargement getRegleChargement() { - return regleChargement; - } - public void setRegleChargement(RegleChargement regleChargement) { - this.regleChargement = regleChargement; - } + /** * va chercher en base les normes et les renvoie sous forme d'un array @@ -80,9 +74,9 @@ public void setRegleChargement(RegleChargement regleChargement) { * @return * @throws ArcException */ - public static List getNormesBase(Connection connexion, String envExecution) { + public static List getNormesBase(Connection connexion, String envExecution) { - List output = new ArrayList() ; + List output = new ArrayList() ; // Récupérer les régles de définition de normes List> normes = new ArrayList<>(); try { @@ -94,7 +88,7 @@ public static List getNormesBase(Connection connexion, String envExecutio //boucle sur les normes for (int i=0; i parser; - public ChargeurCsvDao(Sandbox sandbox, FileAttributes fileAttributes, Norme norme, + public ChargeurCsvDao(Sandbox sandbox, FileAttributesCSV fileAttributes, FileIdCard fileIdCard, ParseFormatRulesOperation parser) { this.sandbox = sandbox; this.fileAttributes = fileAttributes; - this.norme = norme; + this.fileIdCard = fileIdCard; this.parser = parser; } @@ -82,7 +83,7 @@ public void execQueryCopyCsv(InputStream streamContent) throws ArcException { boolean ignoreFirstLine = (parser.getValue(FormatRulesCsv.HEADERS) == null); - String separateur = norme.getRegleChargement().getDelimiter(); + String separateur = fileIdCard.getRegleChargement().getDelimiter(); String quote = parser.getValue(FormatRulesCsv.QUOTE); @@ -103,18 +104,18 @@ public void execQueryCreateContainerWithArcMetadata() throws ArcException { req.append("DROP TABLE IF EXISTS " + ViewEnum.TMP_CHARGEMENT_ARC.getFullName() + ";"); req.append("CREATE TEMPORARY TABLE " + ViewEnum.TMP_CHARGEMENT_ARC.getFullName()); req.append(" AS (SELECT "); - req.append("\n '" + fileAttributes.getFileName() + "'::text collate \"C\" as " + req.append("\n '" + fileIdCard.getFileName() + "'::text collate \"C\" as " + ColumnEnum.ID_SOURCE.getColumnName()); req.append("\n ,id::integer as id"); - req.append("\n ," + fileAttributes.getIntegrationDate() + "::text collate \"C\" as date_integration "); - req.append("\n ,'" + norme.getIdNorme() + "'::text collate \"C\" as id_norme "); - req.append("\n ,'" + norme.getPeriodicite() + "'::text collate \"C\" as periodicite "); - req.append("\n ,'" + fileAttributes.getValidite() + "'::text collate \"C\" as validite "); + req.append("\n ," + fileIdCard.getIntegrationDate() + "::text collate \"C\" as date_integration "); + req.append("\n ,'" + fileIdCard.getIdNorme() + "'::text collate \"C\" as id_norme "); + req.append("\n ,'" + fileIdCard.getPeriodicite() + "'::text collate \"C\" as periodicite "); + req.append("\n ,'" + fileIdCard.getValidite() + "'::text collate \"C\" as validite "); req.append("\n ,0::integer as nombre_colonne"); req.append("\n , "); - for (int i = 0; i < fileAttributes.getHeaders().length; i++) { + for (int i = 0; i < fileAttributes.getHeadersI().length; i++) { req.append("id as " + fileAttributes.getHeadersI()[i] + ", " + fileAttributes.getHeadersV()[i] + ","); } @@ -438,7 +439,7 @@ private List execQuerySelectColumnsFromLoadTable() throws ArcException { public void execQueryBilan(String tableChargementPilTemp, String currentPhase) throws ArcException { StringBuilder requeteBilan = new StringBuilder(); - requeteBilan.append(ApiService.pilotageMarkIdsource(tableChargementPilTemp, fileAttributes.getFileName(), + requeteBilan.append(ApiService.pilotageMarkIdsource(tableChargementPilTemp, fileIdCard.getFileName(), currentPhase, TraitementEtat.OK.toString(), null)); UtilitaireDao.get(0).executeBlock(sandbox.getConnection(), requeteBilan); diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ChargementBrut.java b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ChargementBrut.java index 93466aee2..bee3fb717 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ChargementBrut.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ChargementBrut.java @@ -13,7 +13,8 @@ import fr.insee.arc.core.dataobjects.ArcPreparedStatementBuilder; import fr.insee.arc.core.dataobjects.ColumnEnum; -import fr.insee.arc.core.service.p2chargement.bo.Norme; +import fr.insee.arc.core.service.p2chargement.bo.FileIdCard; +import fr.insee.arc.core.service.p2chargement.bo.NormeRules; import fr.insee.arc.core.util.StaticLoggerDispatcher; import fr.insee.arc.utils.dao.UtilitaireDao; import fr.insee.arc.utils.exception.ArcException; @@ -34,7 +35,7 @@ public class ChargementBrut { private static final Logger LOGGER = LogManager.getLogger(ChargementBrut.class); private Connection connexion; - private List listeNorme; + private List listeNorme; /** Retourne une requête (SELECT) contenant l'idsource et des lignes du fichier. * @param idSource du fichier chargé @@ -84,13 +85,10 @@ private String requeteFichierBrutalement(String idSource, BufferedReader br, int /** Calcule la norme. Retourne (par référence) la norme dans normeOk[0] et la validité dans validiteOk[0]. * @throws IOException * @throws ArcException si aucune norme ou plus d'une norme trouvée */ - public void calculeNormeAndValiditeFichiers(String idSource, InputStream file, Norme[] normeOk, String[] validiteOk) + public void calculeNormeAndValiditeFichiers(InputStream file, FileIdCard normeOk) throws ArcException { StaticLoggerDispatcher.info(LOGGER, "** calculeNormeFichiers **"); - - normeOk[0] = new Norme(); - validiteOk[0]= null; - + int nbBoucle = 0; try(InputStreamReader isr = new InputStreamReader(file); @@ -102,19 +100,22 @@ public void calculeNormeAndValiditeFichiers(String idSource, InputStream file, N // nbBoucle getListeNorme() { + public List getListeNorme() { return listeNorme; } @@ -194,7 +197,7 @@ public List getListeNorme() { * @param listeNorme * the listeNorme to set */ - public void setListeNorme(List listeNorme) { + public void setListeNorme(List listeNorme) { this.listeNorme = listeNorme; } diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ChargeurCSV.java b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ChargeurCSV.java index dda0a65f3..f80744f85 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ChargeurCSV.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ChargeurCSV.java @@ -13,9 +13,9 @@ import fr.insee.arc.core.service.global.bo.Sandbox; import fr.insee.arc.core.service.p2chargement.bo.Delimiters; -import fr.insee.arc.core.service.p2chargement.bo.FileAttributes; +import fr.insee.arc.core.service.p2chargement.bo.FileIdCard; +import fr.insee.arc.core.service.p2chargement.bo.FileAttributesCSV; import fr.insee.arc.core.service.p2chargement.bo.FormatRulesCsv; -import fr.insee.arc.core.service.p2chargement.bo.Norme; import fr.insee.arc.core.service.p2chargement.dao.ChargeurCsvDao; import fr.insee.arc.core.service.p2chargement.thread.ThreadChargementService; import fr.insee.arc.core.util.StaticLoggerDispatcher; @@ -43,24 +43,23 @@ public class ChargeurCSV implements IChargeur { private InputStream streamContent; private Sandbox sandbox; - private FileAttributes fileAttributes; - private Norme norme; + private FileIdCard fileIdCard; + private FileAttributesCSV fileAttributes; + private ParseFormatRulesOperation parser; private ChargeurCsvDao dao; - public ChargeurCSV(ThreadChargementService threadChargementService, String fileName) { + public ChargeurCSV(ThreadChargementService threadChargementService) { this.sandbox = new Sandbox(threadChargementService.getConnexion().getExecutorConnection(), threadChargementService.getEnvExecution()); - this.fileAttributes = new FileAttributes(fileName, threadChargementService.validite); - this.norme = threadChargementService.normeOk; - this.parser = new ParseFormatRulesOperation<>(norme, FormatRulesCsv.class); - - this.dao = new ChargeurCsvDao(this.sandbox, this.fileAttributes, this.norme, this.parser); + this.fileIdCard = threadChargementService.fileIdCard; + this.fileAttributes = new FileAttributesCSV(); + this.parser = new ParseFormatRulesOperation<>(fileIdCard, FormatRulesCsv.class); + this.dao = new ChargeurCsvDao(this.sandbox, this.fileAttributes, this.fileIdCard, this.parser); this.tableChargementPilTemp = threadChargementService.getTableChargementPilTemp(); - this.currentPhase = threadChargementService.getCurrentPhase(); this.streamContent = threadChargementService.filesInputStreamLoad.getTmpInxCSV(); this.streamHeader = threadChargementService.filesInputStreamLoad.getTmpInxChargement(); @@ -87,14 +86,14 @@ private void copyCsvFileToDatabase() throws ArcException { StaticLoggerDispatcher.info(LOGGER, "** CSVtoBase begin **"); StaticLoggerDispatcher.debug(LOGGER, - String.format("contenu delimiter %s", norme.getRegleChargement().getDelimiter())); + String.format("contenu delimiter %s", fileIdCard.getRegleChargement().getDelimiter())); StaticLoggerDispatcher.debug(LOGGER, - String.format("contenu format %s", norme.getRegleChargement().getFormat())); + String.format("contenu format %s", fileIdCard.getRegleChargement().getFormat())); // update delimiter - norme.getRegleChargement() + fileIdCard.getRegleChargement() .setDelimiter(ObjectUtils.firstNonNull( - dao.execQueryEvaluateCharExpression(norme.getRegleChargement().getDelimiter().trim()), + dao.execQueryEvaluateCharExpression(fileIdCard.getRegleChargement().getDelimiter().trim()), Delimiters.DEFAULT_CSV_DELIMITER)); // update quote @@ -123,7 +122,7 @@ private void copyCsvFileToDatabase() throws ArcException { */ private void computeHeaders() throws ArcException { String userDefinedHeaders = parser.getValue(FormatRulesCsv.HEADERS); - String csvDelimiter = norme.getRegleChargement().getDelimiter(); + String csvDelimiter = fileIdCard.getRegleChargement().getDelimiter(); // si le headers n'est pas spécifié, alors on le cherche dans le fichier en // premier ligne @@ -141,7 +140,7 @@ private void computeHeaders() throws ArcException { } } catch (IOException fileReadException) { throw new ArcException(fileReadException, ArcExceptionMessage.FILE_READ_FAILED, - fileAttributes.getFileName()); + fileIdCard.getFileName()); } } else { String[] headers = Format.tokenizeAndTrim(userDefinedHeaders, Delimiters.HEADERS_DELIMITER); @@ -150,7 +149,6 @@ private void computeHeaders() throws ArcException { } private void registerHeaders(String[] headers) { - this.fileAttributes.setHeaders(headers); this.fileAttributes.setHeadersI(Format.toBdId(headers)); this.fileAttributes.setHeadersV(Format.toBdVal(headers)); } @@ -172,7 +170,7 @@ private void importCsvDataToTable() throws ArcException { } } catch (IOException fileReadException) { throw new ArcException(fileReadException, ArcExceptionMessage.FILE_READ_FAILED, - this.fileAttributes.getFileName()); + this.fileIdCard.getFileName()); } } diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ChargeurClefValeur.java b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ChargeurClefValeur.java index 2c5c3da98..903634a2f 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ChargeurClefValeur.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ChargeurClefValeur.java @@ -17,7 +17,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import fr.insee.arc.core.service.p2chargement.bo.Norme; +import fr.insee.arc.core.service.p2chargement.bo.FileIdCard; import fr.insee.arc.core.service.p2chargement.thread.ThreadChargementService; import fr.insee.arc.core.service.p2chargement.xmlhandler.ArbreFormat; import fr.insee.arc.core.util.StaticLoggerDispatcher; @@ -35,8 +35,7 @@ public class ChargeurClefValeur implements IChargeur { private static final Logger LOGGER = LogManager.getLogger(ChargeurClefValeur.class); - private Norme normeOk; - private String separateur = ","; + private FileIdCard fileIdCard; private String envExecution; private String idSource; private PipedOutputStream outputStream; @@ -44,13 +43,12 @@ public class ChargeurClefValeur implements IChargeur { private InputStream tmpInxChargement; private ChargeurXml chargeurXml; - public ChargeurClefValeur(ThreadChargementService threadChargementService, String fileName) { + public ChargeurClefValeur(ThreadChargementService threadChargementService) { super(); this.tmpInxChargement = threadChargementService.filesInputStreamLoad.getTmpInxChargement(); - this.normeOk = threadChargementService.normeOk; - this.separateur = this.normeOk.getRegleChargement().getDelimiter(); - this.chargeurXml = new ChargeurXml(threadChargementService, fileName); + this.fileIdCard = threadChargementService.fileIdCard; + this.chargeurXml = new ChargeurXml(threadChargementService); this.envExecution = threadChargementService.getEnvExecution(); } @@ -62,7 +60,7 @@ public void run() { try { // On lit le fichier format et on en retire une map (rubrique, père) - ArbreFormat arbreFormat = new ArbreFormat(getNormeOk()); + ArbreFormat arbreFormat = new ArbreFormat(getFileIdCard()); // Lire le fichier d'entrée et le convertir en fichier XML. Pour ne pas écrire dans la mémoire on transforme un // stream en un autre stream @@ -178,8 +176,8 @@ private List initialisationOutputStream(Map arbreFormat, ecrireXML("\n"); - String rubrique = ManipString.substringBeforeFirst(ligne, separateur); - String donnee = ManipString.substringAfterFirst(ligne, separateur); + String rubrique = ManipString.substringBeforeFirst(ligne, fileIdCard.getRegleChargement().getDelimiter()); + String donnee = ManipString.substringAfterFirst(ligne, fileIdCard.getRegleChargement().getDelimiter()); // On retire les quotes de début et fin de manière violente donnee = donnee.substring(1, donnee.length() - 1); @@ -236,8 +234,8 @@ private List lectureLigne(Map arbreFormat, String pere; String rubriqueCourante; - rubrique = ManipString.substringBeforeFirst(ligne, separateur); - donnee = ManipString.substringAfterFirst(ligne, separateur); + rubrique = ManipString.substringBeforeFirst(ligne, fileIdCard.getRegleChargement().getDelimiter()); + donnee = ManipString.substringAfterFirst(ligne, fileIdCard.getRegleChargement().getDelimiter()); // On retire les quotes de début et fin de manière violente donnee = donnee.substring(1, donnee.length() - 1); @@ -345,15 +343,18 @@ private void ecrireXML(String donnee) throws ArcException { } } - public Norme getNormeOk() { - return normeOk; - } + + + + public FileIdCard getFileIdCard() { + return fileIdCard; + } - public void setNormeOk(Norme normeOk) { - this.normeOk = normeOk; - } + public void setFileIdCard(FileIdCard fileIdCard) { + this.fileIdCard = fileIdCard; + } - public String getEnvExecution() { + public String getEnvExecution() { return envExecution; } diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ChargeurXml.java b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ChargeurXml.java index 8ff372e1f..e575686ec 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ChargeurXml.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ChargeurXml.java @@ -15,7 +15,8 @@ import fr.insee.arc.core.dataobjects.ColumnEnum; import fr.insee.arc.core.model.TraitementEtat; import fr.insee.arc.core.service.global.ApiService; -import fr.insee.arc.core.service.p2chargement.bo.Norme; +import fr.insee.arc.core.service.p2chargement.bo.FileIdCard; +import fr.insee.arc.core.service.p2chargement.bo.NormeRules; import fr.insee.arc.core.service.p2chargement.thread.ThreadChargementService; import fr.insee.arc.core.service.p2chargement.xmlhandler.XMLHandlerCharger4; import fr.insee.arc.core.util.StaticLoggerDispatcher; @@ -36,12 +37,10 @@ */ public class ChargeurXml implements IChargeur { private static final Logger LOGGER = LogManager.getLogger(ChargeurXml.class); - private String fileName; + private FileIdCard fileIdCard; private Connection connexion; private String tableChargementPilTemp; private String currentPhase; - private Norme norme; - private String validite; private InputStream f; // temporary table where data will be loaded by the XML SAX engine @@ -63,15 +62,13 @@ public class ChargeurXml implements IChargeur { * @param threadChargementService * @param fileName */ - public ChargeurXml(ThreadChargementService threadChargementService, String fileName) { - this.fileName = fileName; + public ChargeurXml(ThreadChargementService threadChargementService) { + this.fileIdCard = threadChargementService.fileIdCard; this.connexion = threadChargementService.getConnexion().getExecutorConnection(); this.tableTempA = threadChargementService.getTableTempA(); this.tableChargementPilTemp = threadChargementService.getTableChargementPilTemp(); this.currentPhase = threadChargementService.getCurrentPhase(); this.f = threadChargementService.filesInputStreamLoad.getTmpInxChargement(); - this.norme = threadChargementService.normeOk; - this.validite = threadChargementService.validite; } /** @@ -136,7 +133,7 @@ public void initialisation() throws ArcException { @Override public void finalisation() { StringBuilder requeteBilan = new StringBuilder(); - requeteBilan.append(ApiService.pilotageMarkIdsource(this.tableChargementPilTemp, fileName, this.currentPhase, + requeteBilan.append(ApiService.pilotageMarkIdsource(this.tableChargementPilTemp, fileIdCard.getFileName(), this.currentPhase, TraitementEtat.OK.toString(), rapport, this.jointure)); try { @@ -152,7 +149,7 @@ public void execution() throws ArcException { java.util.Date beginDate = new java.util.Date(); // Création de la table de stockage - XMLHandlerCharger4 handler = new XMLHandlerCharger4(connexion, fileName, norme, validite, this.tableTempA, + XMLHandlerCharger4 handler = new XMLHandlerCharger4(connexion, fileIdCard, this.tableTempA, this.tempTableAColumnsLongName, this.tempTableAColumnsShortName); // appel du parser et gestion d'erreur @@ -161,7 +158,7 @@ public void execution() throws ArcException { saxParser.parse(f, handler); } catch (ParserConfigurationException | SAXException | IOException e) { ArcException businessException = new ArcException(e, ArcExceptionMessage.XML_SAX_PARSING_FAILED, - this.fileName).logMessageException(); + fileIdCard.getFileName()).logMessageException(); rapport = businessException.getMessage().replace("'", "''"); throw businessException; } diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ChargeurXmlComplexe.java b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ChargeurXmlComplexe.java index 6f56e461f..6099572bf 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ChargeurXmlComplexe.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ChargeurXmlComplexe.java @@ -5,8 +5,6 @@ import java.sql.Connection; import java.util.ArrayList; import java.util.Arrays; -import java.util.List; -import java.util.Map; import javax.xml.parsers.ParserConfigurationException; import javax.xml.parsers.SAXParser; @@ -16,12 +14,9 @@ import org.xml.sax.SAXException; import fr.insee.arc.core.dataobjects.ColumnEnum; -import fr.insee.arc.core.dataobjects.ViewEnum; import fr.insee.arc.core.model.TraitementEtat; import fr.insee.arc.core.service.global.ApiService; -import fr.insee.arc.core.service.global.bo.NormeFichier; -import fr.insee.arc.core.service.global.dao.RulesOperations; -import fr.insee.arc.core.service.p2chargement.bo.Norme; +import fr.insee.arc.core.service.p2chargement.bo.FileIdCard; import fr.insee.arc.core.service.p2chargement.thread.ThreadChargementService; import fr.insee.arc.core.service.p2chargement.xmlhandler.XMLComplexeHandlerCharger; import fr.insee.arc.core.util.StaticLoggerDispatcher; @@ -30,7 +25,6 @@ import fr.insee.arc.utils.exception.ArcExceptionMessage; import fr.insee.arc.utils.textUtils.FastList; import fr.insee.arc.utils.utils.FormatSQL; -import fr.insee.arc.utils.utils.LoggerHelper; import fr.insee.arc.utils.utils.Pair; import fr.insee.arc.utils.utils.SecuredSaxParser; @@ -43,13 +37,10 @@ */ public class ChargeurXmlComplexe implements IChargeur { private static final Logger LOGGER = LogManager.getLogger(ChargeurXmlComplexe.class); - private String fileName; private Connection connexion; private String tableChargementPilTemp; - private String envExecution; private String currentPhase; - private Norme norme; - private String validite; + private FileIdCard fileIdCard; private InputStream f; private ArrayList> format; @@ -64,29 +55,22 @@ public class ChargeurXmlComplexe implements IChargeur { "text collate \"C\"", "text collate \"C\"", "text collate \"C\"", "text collate \"C\"")); private String rapport; - public String jointure; + private String jointure; - public ChargeurXmlComplexe(ThreadChargementService threadChargementService, String fileName) { - this.fileName = fileName; + public ChargeurXmlComplexe(ThreadChargementService threadChargementService) { this.connexion = threadChargementService.getConnexion().getExecutorConnection(); this.tableTempA = threadChargementService.getTableTempA(); this.tableChargementPilTemp = threadChargementService.getTableChargementPilTemp(); this.currentPhase = threadChargementService.getCurrentPhase(); this.f = threadChargementService.filesInputStreamLoad.getTmpInxChargement(); - this.norme = threadChargementService.normeOk; - this.validite = threadChargementService.validite; - this.envExecution = threadChargementService.getEnvExecution(); + this.fileIdCard = threadChargementService.fileIdCard; } - public ChargeurXmlComplexe(Connection connexion, String fileName, InputStream f, String tableOut, String norme, - String periodicite, String validite, String envExecution) { - this.fileName = fileName; + public ChargeurXmlComplexe(Connection connexion, FileIdCard fileIdCard, InputStream f, String tableOut) { + this.fileIdCard = fileIdCard; this.connexion = connexion; this.tableTempA = tableOut; - this.norme = new Norme(norme, periodicite, null, null); - this.validite = validite; this.f = f; - this.envExecution = envExecution; } /** @@ -105,13 +89,9 @@ public void initialisation() throws ArcException { java.util.Date beginDate = new java.util.Date(); - NormeFichier normeFichier = new NormeFichier(this.norme.getIdNorme(), validite, this.norme.getPeriodicite()); this.format = new ArrayList<>(); - - Map> regle = RulesOperations.getBean(this.connexion, - RulesOperations.getRegles(ViewEnum.CHARGEMENT_REGLE.getFullName(this.envExecution), normeFichier)); - if (regle.get("format").get(0) != null) { - for (String rule : regle.get("format").get(0).split("\n")) { + if (this.fileIdCard.getRegleChargement().getFormat() != null) { + for (String rule : this.fileIdCard.getRegleChargement().getFormat().split("\n")) { this.format.add(new Pair<>(rule.split(",")[0].trim(), rule.split(",")[1].trim())); } } @@ -157,7 +137,7 @@ public void initialisation() throws ArcException { @Override public void finalisation() throws ArcException { StringBuilder requeteBilan = new StringBuilder(); - requeteBilan.append(ApiService.pilotageMarkIdsource(this.tableChargementPilTemp, fileName, + requeteBilan.append(ApiService.pilotageMarkIdsource(this.tableChargementPilTemp, fileIdCard.getFileName(), this.currentPhase, TraitementEtat.OK.toString(), rapport, this.jointure)); UtilitaireDao.get(0).executeBlock(this.connexion, requeteBilan); } @@ -168,7 +148,7 @@ public void execution() throws ArcException { java.util.Date beginDate = new java.util.Date(); // Création de la table de stockage - XMLComplexeHandlerCharger handler = new XMLComplexeHandlerCharger(connexion, fileName, norme, validite, this.tableTempA, + XMLComplexeHandlerCharger handler = new XMLComplexeHandlerCharger(connexion, fileIdCard, this.tableTempA, this.tempTableAColumnsLongName, this.tempTableAColumnsShortName, format); // appel du parser et gestion d'erreur try { @@ -176,7 +156,7 @@ public void execution() throws ArcException { saxParser.parse(f, handler); } catch (ParserConfigurationException | SAXException | IOException e) { ArcException businessException = new ArcException(e, ArcExceptionMessage.XML_SAX_PARSING_FAILED, - this.fileName).logMessageException(); + this.fileIdCard.getFileName()).logMessageException(); rapport = businessException.getMessage().replace("'", "''"); throw businessException; } @@ -210,4 +190,9 @@ public void setF(InputStream f) { this.f = f; } + public String getJointure() { + return jointure; + } + + } diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ParseFormatRulesOperation.java b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ParseFormatRulesOperation.java index 36e9c103f..152c31bcc 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ParseFormatRulesOperation.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/engine/ParseFormatRulesOperation.java @@ -6,20 +6,20 @@ import java.util.List; import java.util.Map; +import fr.insee.arc.core.service.p2chargement.bo.FileIdCard; import fr.insee.arc.core.service.p2chargement.bo.IParseFormatRules; -import fr.insee.arc.core.service.p2chargement.bo.Norme; import fr.insee.arc.utils.utils.ManipString; public class ParseFormatRulesOperation { - public ParseFormatRulesOperation(Norme norme, Class type) { + public ParseFormatRulesOperation(FileIdCard fileIdCard, Class type) { super(); - this.norme = norme; + this.fileIdCard = fileIdCard; this.parseResult = new HashMap<>(); this.type = type; } - private Norme norme; + private FileIdCard fileIdCard; private Class type; private Map> parseResult; @@ -38,7 +38,7 @@ public void setValue(T t, String value) { public void parseFormatRules() { - String formatRules = this.norme.getRegleChargement().getFormat(); + String formatRules = this.fileIdCard.getRegleChargement().getFormat(); if (formatRules == null || formatRules.isBlank()) { return; } diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/factory/ChargeurFactory.java b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/factory/ChargeurFactory.java index db82a9776..4d8137420 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/factory/ChargeurFactory.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/factory/ChargeurFactory.java @@ -25,15 +25,15 @@ public class ChargeurFactory { private static final Logger LOGGER = LogManager.getLogger(ChargeurFactory.class); - public ChargeurFactory(ThreadChargementService threadChargementService, String fileName) { + public ChargeurFactory(ThreadChargementService threadChargementService) { this.map.put(TypeChargement.XML, - new ChargeurXml(threadChargementService,fileName)); + new ChargeurXml(threadChargementService)); this.map.put(TypeChargement.XML_COMPLEXE, - new ChargeurXmlComplexe(threadChargementService,fileName)); + new ChargeurXmlComplexe(threadChargementService)); this.map.put(TypeChargement.CLEF_VALEUR, - new ChargeurClefValeur(threadChargementService,fileName)); + new ChargeurClefValeur(threadChargementService)); this.map.put(TypeChargement.PLAT, - new ChargeurCSV(threadChargementService,fileName)); + new ChargeurCSV(threadChargementService)); } diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/thread/ThreadChargementService.java b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/thread/ThreadChargementService.java index efd51685f..63b0b92b8 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/thread/ThreadChargementService.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/thread/ThreadChargementService.java @@ -2,6 +2,8 @@ import java.io.File; import java.io.IOException; +import java.util.List; +import java.util.Map; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -14,6 +16,7 @@ import fr.insee.arc.core.service.global.dao.DatabaseConnexionConfiguration; import fr.insee.arc.core.service.global.dao.HashFileNameConversion; import fr.insee.arc.core.service.global.dao.PilotageOperations; +import fr.insee.arc.core.service.global.dao.RulesOperations; import fr.insee.arc.core.service.global.dao.TableNaming; import fr.insee.arc.core.service.global.dao.TableOperations; import fr.insee.arc.core.service.global.dao.ThreadOperations; @@ -23,8 +26,9 @@ import fr.insee.arc.core.service.p2chargement.archiveloader.ArchiveChargerFactory; import fr.insee.arc.core.service.p2chargement.archiveloader.FilesInputStreamLoad; import fr.insee.arc.core.service.p2chargement.archiveloader.IArchiveFileLoader; -import fr.insee.arc.core.service.p2chargement.bo.Norme; -import fr.insee.arc.core.service.p2chargement.bo.RegleChargement; +import fr.insee.arc.core.service.p2chargement.bo.FileIdCard; +import fr.insee.arc.core.service.p2chargement.bo.NormeRules; +import fr.insee.arc.core.service.p2chargement.bo.FileIdCardChargement; import fr.insee.arc.core.service.p2chargement.engine.ChargementBrut; import fr.insee.arc.core.service.p2chargement.engine.IChargeur; import fr.insee.arc.core.service.p2chargement.factory.ChargeurFactory; @@ -49,18 +53,15 @@ public class ThreadChargementService extends ApiChargementService implements Run private Thread t; - private String container; private String tableChargementPilTemp; private ThreadOperations arcThreadGenericDao; - public String validite; - public FilesInputStreamLoad filesInputStreamLoad; - public Norme normeOk; + public FileIdCard fileIdCard; protected String tableChargementOK; @@ -253,31 +254,27 @@ private IChargeur chooseLoader() throws ArcException { // Si on a pas 1 seule norme alors le fichier est en erreur ChargementBrut chgrBrtl = new ChargementBrut(); chgrBrtl.setConnexion(getConnexion().getExecutorConnection()); - chgrBrtl.setListeNorme(listeNorme); - + chgrBrtl.setListeNorme(listeNorme); // Stockage dans des tableaux pour passage par référence - Norme[] n = new Norme[1]; - String[] v = new String[1]; - + + this.fileIdCard = new FileIdCard(this.idSource); + try { - chgrBrtl.calculeNormeAndValiditeFichiers(this.idSource, this.filesInputStreamLoad.getTmpInxNormage(), n, v); + chgrBrtl.calculeNormeAndValiditeFichiers(this.filesInputStreamLoad.getTmpInxNormage(), this.fileIdCard); } catch (Exception e) { LoggerHelper.error(LOGGER, e); throw e; } finally { - majPilotage(this.idSource, n[0], v[0]); + majPilotage(this.idSource, this.fileIdCard); } - this.normeOk = n[0]; - this.validite = v[0]; - // Quel type de fichier ? - normeOk = calculerTypeFichier(normeOk); + calculerTypeFichier(); - ChargeurFactory chargeurFactory = new ChargeurFactory(this, this.idSource); + ChargeurFactory chargeurFactory = new ChargeurFactory(this); - return chargeurFactory.getChargeur(this.normeOk.getRegleChargement().getTypeChargement()); + return chargeurFactory.getChargeur(this.fileIdCard.getRegleChargement().getTypeChargement()); } /** @@ -290,22 +287,24 @@ private IChargeur chooseLoader() throws ArcException { * @throws ArcException * @throws ArcException si aucune règle n'est trouvée */ - private Norme calculerTypeFichier(Norme norme) throws ArcException { - - ArcPreparedStatementBuilder requete = new ArcPreparedStatementBuilder(); - requete.append("SELECT type_fichier, delimiter, format ").append(" FROM " + ViewEnum.CHARGEMENT_REGLE.getFullName(this.getEnvExecution())) - .append(" WHERE id_norme =" + requete.quoteText(norme.getIdNorme()) + ";"); - - GenericBean g = new GenericBean( - UtilitaireDao.get(0).executeRequest(this.getConnexion().getExecutorConnection(), requete)); - if (g.mapContent().isEmpty()) { - throw new ArcException(ArcExceptionMessage.LOAD_RULES_NOT_FOUND, norme.getIdNorme()); + private void calculerTypeFichier() throws ArcException { + Map> regle = RulesOperations.getBean(this.getConnexion().getExecutorConnection(), + RulesOperations.getRegles(ViewEnum.CHARGEMENT_REGLE.getFullName(this.envExecution), this.fileIdCard)); + + if (regle.get(ColumnEnum.TYPE_FICHIER.getColumnName()).isEmpty()) + { + throw new ArcException(ArcExceptionMessage.LOAD_RULES_NOT_FOUND, fileIdCard.getIdNorme()); } - - norme.setRegleChargement(new RegleChargement(TypeChargement.getEnum(g.content.get(0).get(0)), - g.content.get(0).get(1), g.content.get(0).get(2))); - - return norme; + + if (regle.get(ColumnEnum.TYPE_FICHIER.getColumnName()).size()>1) + { + throw new ArcException(ArcExceptionMessage.LOAD_RULES_NOT_FOUND, fileIdCard.getIdNorme()); + } + + fileIdCard.setRegleChargement(new FileIdCardChargement( + TypeChargement.getEnum(regle.get(ColumnEnum.TYPE_FICHIER.getColumnName()).get(0)), + regle.get(ColumnEnum.DELIMITER.getColumnName()).get(0) + , regle.get(ColumnEnum.FORMAT.getColumnName()).get(0))); } /** @@ -338,7 +337,7 @@ private String insertionFinale(String tableName, String idSource) throws ArcExce * @throws ArcException */ - private boolean majPilotage(String idSource, Norme normeOk, String validite) throws ArcException { + private boolean majPilotage(String idSource, FileIdCard fileIdCard) throws ArcException { boolean erreur = false; StaticLoggerDispatcher.info(LOGGER, "Mettre à jour la table de pilotage"); java.util.Date beginDate = new java.util.Date(); @@ -347,16 +346,16 @@ private boolean majPilotage(String idSource, Norme normeOk, String validite) thr bloc3.append("UPDATE " + this.tableChargementPilTemp + " a \n"); bloc3.append("SET\n"); - if (normeOk.getIdNorme() == null) { + if (fileIdCard.getIdNorme() == null) { bloc3.append(" id_norme='" + TraitementRapport.NORMAGE_NO_NORME + "' "); bloc3.append(", validite= '" + TraitementRapport.NORMAGE_NO_DATE + "' "); bloc3.append(", periodicite='" + TraitementRapport.NORMAGE_NO_NORME + "' "); bloc3.append(", etat_traitement='{" + TraitementEtat.KO + "}' "); } else { - bloc3.append(" id_norme='" + normeOk.getIdNorme() + "' \n"); - bloc3.append(", validite='" + validite + "' \n"); - bloc3.append(", periodicite='" + normeOk.getPeriodicite() + "' \n"); + bloc3.append(" id_norme='" + fileIdCard.getIdNorme() + "' \n"); + bloc3.append(", validite='" + fileIdCard.getValidite() + "' \n"); + bloc3.append(", periodicite='" + fileIdCard.getPeriodicite() + "' \n"); } bloc3.append("where " + ColumnEnum.ID_SOURCE.getColumnName() + "='" + idSource + "' AND phase_traitement='" @@ -399,4 +398,14 @@ public void setTableTempA(String tableTempA) { this.tableTempA = tableTempA; } + public FileIdCard getFileIdCard() { + return fileIdCard; + } + + public void setFileIdCard(FileIdCard fileIdCard) { + this.fileIdCard = fileIdCard; + } + + + } \ No newline at end of file diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/xmlhandler/ArbreFormat.java b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/xmlhandler/ArbreFormat.java index 25048ec20..e6fca186f 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/xmlhandler/ArbreFormat.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/xmlhandler/ArbreFormat.java @@ -14,7 +14,8 @@ import org.xml.sax.SAXException; import fr.insee.arc.core.model.XMLConstant; -import fr.insee.arc.core.service.p2chargement.bo.Norme; +import fr.insee.arc.core.service.p2chargement.bo.FileIdCard; +import fr.insee.arc.core.service.p2chargement.bo.NormeRules; import fr.insee.arc.utils.exception.ArcException; import fr.insee.arc.utils.exception.ArcExceptionMessage; import fr.insee.arc.utils.utils.SecuredSaxParser; @@ -41,7 +42,7 @@ public class ArbreFormat { //éléments intermédiaire private List branches = new ArrayList<>(); - public ArbreFormat(Norme aNorme) throws ArcException { + public ArbreFormat(FileIdCard fileIdCard) throws ArcException { super(); try { @@ -50,7 +51,7 @@ public ArbreFormat(Norme aNorme) throws ArcException { SAXParser saxParser = SecuredSaxParser.buildSecuredSaxParser(); FormatFichierHandler formatHandler = new FormatFichierHandler(); - saxParser.parse(new InputSource(new StringReader(aNorme.getRegleChargement().getFormat())), formatHandler); + saxParser.parse(new InputSource(new StringReader(fileIdCard.getRegleChargement().getFormat())), formatHandler); this.arbreHierachieDuFichier = formatHandler.getArbre(); calculerFeuilles(); diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/xmlhandler/XMLComplexeHandlerCharger.java b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/xmlhandler/XMLComplexeHandlerCharger.java index cb9e6e45f..e5f9e301e 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/xmlhandler/XMLComplexeHandlerCharger.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/xmlhandler/XMLComplexeHandlerCharger.java @@ -2,7 +2,6 @@ import java.sql.Connection; import java.util.ArrayList; -import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -14,8 +13,7 @@ import org.xml.sax.SAXParseException; import fr.insee.arc.core.dataobjects.ColumnEnum; -import fr.insee.arc.core.service.global.dao.DateConversion; -import fr.insee.arc.core.service.p2chargement.bo.Norme; +import fr.insee.arc.core.service.p2chargement.bo.FileIdCard; import fr.insee.arc.utils.dataobjects.TypeEnum; import fr.insee.arc.utils.exception.ArcException; import fr.insee.arc.utils.format.Format; @@ -31,14 +29,12 @@ public class XMLComplexeHandlerCharger extends org.xml.sax.helpers.DefaultHandler { private static final Logger LOGGER = LogManager.getLogger(XMLHandlerCharger4.class); - public XMLComplexeHandlerCharger(Connection connexion, String fileName, Norme normeCourante, String validite, + public XMLComplexeHandlerCharger(Connection connexion, FileIdCard fileIdCard, String tempTableA, FastList tempTableAColumnsLongName, FastList tempTableAColumnsShortName, ArrayList> format) { super(); this.connexion = connexion; - this.fileName = fileName; - this.normeCourante = normeCourante; - this.validite = validite; + this.fileIdCard = fileIdCard; this.tempTableA = tempTableA; this.tempTableAColumnsLongName = tempTableAColumnsLongName; this.tempTableAColumnsShortName = tempTableAColumnsShortName; @@ -50,10 +46,8 @@ public XMLComplexeHandlerCharger(Connection connexion, String fileName, Norme no // input + private FileIdCard fileIdCard; private Connection connexion; - private String fileName; - private Norme normeCourante; - private String validite; private String tempTableA; private FastList tempTableAColumnsLongName; private FastList tempTableAColumnsShortName; @@ -118,9 +112,6 @@ public XMLComplexeHandlerCharger(Connection connexion, String fileName, Norme no private Map requetes = new HashMap<>(); private int requetesLength = 0; - // initialize the integration date with current - private final String integrationDate = DateConversion.queryDateConversion(new Date()); - // format to rename column with format rules public ArrayList> format; @@ -152,7 +143,7 @@ public void characters(char[] caracteres, int debut, int longueur) { */ @Override public void endDocument() throws SAXParseException { - insertQueryBuilder(this.tempTableA, this.fileName, this.lineCols, this.lineIds, this.lineValues); + insertQueryBuilder(this.tempTableA, this.fileIdCard.getFileName(), this.lineCols, this.lineIds, this.lineValues); StringBuilder requete = new StringBuilder(computeFinalQuery()); renameColumns(requete); @@ -266,7 +257,7 @@ public void endElement(String uri, String localName, String qName) throws SAXPar if (this.lineCols.indexOf(this.allCols.indexOf(this.closedTag)) < this.lineCols.size() - 1) { // réalisation de l'insertion - insertQueryBuilder(this.tempTableA, this.fileName, this.lineCols.subList(0, this.lineCols.size() - 1), + insertQueryBuilder(this.tempTableA, this.fileIdCard.getFileName(), this.lineCols.subList(0, this.lineCols.size() - 1), this.lineIds.subList(0, this.lineCols.size() - 1), this.lineValues.subList(0, this.lineCols.size() - 1)); @@ -387,7 +378,7 @@ public void startElement(String uri, String localName, String qName, Attributes if (this.treeStackFatherLag.indexOf(this.father) >= 0 && this.leafStatus == false) { - insertQueryBuilder(this.tempTableA, this.fileName, this.lineCols, this.lineIds, this.lineValues); + insertQueryBuilder(this.tempTableA, this.fileIdCard.getFileName(), this.lineCols, this.lineIds, this.lineValues); if (this.requetesLength > FormatSQL.TAILLE_MAXIMAL_BLOC_SQL) { @@ -499,9 +490,9 @@ private void insertQueryBuilder(String tempTableI, String fileName, List tempTableAColumnsLongName, FastList tempTableAColumnsShortName) { super(); this.connexion = connexion; - this.fileName = fileName; - this.normeCourante = normeCourante; - this.validite = validite; + this.fileIdCard = fileIdCard; this.tempTableA = tempTableA; this.tempTableAColumnsLongName = tempTableAColumnsLongName; this.tempTableAColumnsShortName = tempTableAColumnsShortName; @@ -44,9 +42,7 @@ public XMLHandlerCharger4(Connection connexion, String fileName, Norme normeCour // input private Connection connexion; - private String fileName; - private Norme normeCourante; - private String validite; + private FileIdCard fileIdCard; private String tempTableA; private FastList tempTableAColumnsLongName; private FastList tempTableAColumnsShortName; @@ -128,7 +124,7 @@ public void characters(char[] caracteres, int debut, int longueur) { */ @Override public void endDocument() throws SAXParseException { - insertQueryBuilder(this.tempTableA, this.fileName, this.lineCols, this.lineIds, this.lineValues); + insertQueryBuilder(this.tempTableA, this.fileIdCard.getFileName(), this.lineCols, this.lineIds, this.lineValues); StringBuilder requete = new StringBuilder(computeFinalQuery()); renameColumns(requete); @@ -183,7 +179,7 @@ public void endElement(String uri, String localName, String qName) throws SAXPar if (this.lineCols.indexOf(this.allCols.indexOf(this.closedTag)) < this.lineCols.size() - 1) { // réalisation de l'insertion - insertQueryBuilder(this.tempTableA, this.fileName, this.lineCols.subList(0, this.lineCols.size() - 1), + insertQueryBuilder(this.tempTableA, this.fileIdCard.getFileName(), this.lineCols.subList(0, this.lineCols.size() - 1), this.lineIds.subList(0, this.lineCols.size() - 1), this.lineValues.subList(0, this.lineCols.size() - 1)); @@ -284,7 +280,7 @@ public void startElement(String uri, String localName, String qName, Attributes if (this.treeStackFatherLag.indexOf(this.father) >= 0 && this.leafStatus == false) { - insertQueryBuilder(this.tempTableA, this.fileName, this.lineCols, this.lineIds, this.lineValues); + insertQueryBuilder(this.tempTableA, this.fileIdCard.getFileName(), this.lineCols, this.lineIds, this.lineValues); if (this.requetesLength > FormatSQL.TAILLE_MAXIMAL_BLOC_SQL) { @@ -397,8 +393,8 @@ private void insertQueryBuilder(String tempTableI, String fileName, List tempTableAColumnsLongName = new FastList<>(Arrays.asList( + ColumnEnum.ID_SOURCE + , ColumnEnum.ID_SAX + , ColumnEnum.DATE_INTEGRATION + , ColumnEnum.ID_NORME + , ColumnEnum.PERIODICITE + , ColumnEnum.VALIDITE)); + + public static final FastList tempTableAColumnsShortName = new FastList<>( + Arrays.asList("m0", "m1", "m2", "m3", "m4", "m5")); + + /** + * Permet de générer la requête SQL pour le normage + * Pas de serialization car demande métier de la MOA de voir et de pouvoir exécuter la requete générée + * @return + */ + public static String buildQueryJointureXML(String integrationDate, Map tree, Map colDist, FastList allCols, Map colData) { + + // construction de la requete de jointure + StringBuilder req = new StringBuilder(); + + int[][] arr = TreeFunctions.getTreeArrayByDistance(tree, colDist); + StringBuilder reqCreate = new StringBuilder(" \n"); + + StringBuilder reqInsert = new StringBuilder(); + reqInsert.append(" INSERT INTO {table_destination} (id," + ColumnEnum.ID_SOURCE.getColumnName() + + ",date_integration,id_norme,validite,periodicite"); + + StringBuilder reqSelect = new StringBuilder(); + reqSelect.append("\n SELECT row_number() over (), ww.* FROM ("); + reqSelect.append("\n SELECT '{nom_fichier}'," + integrationDate + ",'{id_norme}','{validite}','{periodicite}'"); + + StringBuilder reqFrom = new StringBuilder(); + + int d = 0; + + for (int i = 0; i < arr.length; i++) { + + // pour chaque noeud + + if (arr[i][2] == 1) { + + String leaf = TreeFunctions.getLeafs(arr[i][1], arr, colData, allCols); + + // créer les vues + String leafMax = TreeFunctions.getLeafsMax(arr[i][1], arr, colData, allCols); + reqCreate.append("CREATE TEMPORARY TABLE t_" + allCols.get(arr[i][1]) + " as (select i_" + + allCols.get(arr[i][1]) + " as m_" + allCols.get(arr[i][1]) + " "); + if (arr[i][0] >= 0) { + reqCreate.append( + ", i_" + allCols.get(arr[i][0]) + " as i_" + allCols.get(arr[i][0]) + " "); + } + reqCreate.append(TreeFunctions.getLeafsSpace(arr[i][1], arr, colData, allCols)); + reqCreate.append(" FROM (SELECT i_" + allCols.get(arr[i][1]) + " "); + reqCreate.append(leafMax); + reqCreate.append(" FROM {table_source} where i_" + allCols.get(arr[i][1]) + + " is not null group by i_" + allCols.get(arr[i][1]) + ") a "); + if (arr[i][0] >= 0) { + reqCreate.append(" , (SELECT DISTINCT i_" + allCols.get(arr[i][1]) + " as pivot, i_" + + allCols.get(arr[i][0]) + " FROM {table_source} where i_" + + allCols.get(arr[i][1]) + " is not null) b "); + reqCreate.append(" where a.i_" + allCols.get(arr[i][1]) + " = b.pivot "); + } + reqCreate.append("); \n"); + + // la table vide faite a partir de la table du bloc; ca permet de faire + // simplement des jointures externe avec vide dedans + reqCreate.append("CREATE TEMPORARY TABLE t_" + allCols.get(arr[i][1]) + + "_null as (select * from t_" + allCols.get(arr[i][1]) + " where false); \n"); + + // générer la clause select + reqInsert.append(",i_" + allCols.get(arr[i][1]) + leaf); + reqSelect.append(",m_" + allCols.get(arr[i][1]) + leaf); + + // générer la clause from + + if (arr[i][0] == -1) { + reqFrom.append("t_" + allCols.get(arr[i][1])); + } else { + + if (d != arr[i][3] && d > 0) { + reqFrom.append("\n ) "); + reqFrom.insert(0, "\n ("); + } + + reqFrom.append( + "\n left join t_" + allCols.get(arr[i][1]) + " on m_" + allCols.get(arr[i][0]) + + "=t_" + allCols.get(arr[i][1]) + ".i_" + allCols.get(arr[i][0])); + } + + d = arr[i][3]; + } + } + + reqInsert.append("\n )"); + + reqFrom.insert(0, "\n FROM "); + reqFrom.append("\n WHERE true ) ww "); + + req.append(reqCreate); + req.append(reqInsert); + req.append(reqSelect); + req.append(reqFrom); + + return FormatSQL.quoteText(req.toString()); + + } + + +} diff --git a/arc-utils/src/main/java/fr/insee/arc/utils/exception/ArcExceptionMessage.java b/arc-utils/src/main/java/fr/insee/arc/utils/exception/ArcExceptionMessage.java index 0bd8ca700..d4293557c 100644 --- a/arc-utils/src/main/java/fr/insee/arc/utils/exception/ArcExceptionMessage.java +++ b/arc-utils/src/main/java/fr/insee/arc/utils/exception/ArcExceptionMessage.java @@ -35,6 +35,7 @@ public enum ArcExceptionMessage { LOAD_PARALLEL_INSERT_THREAD_FAILED("L'insertion en parallele des données a échoué"), LOAD_KEYVALUE_VAR_NOT_EXISTS_IN_FORMAT_RULES("La rubrique fille %s n'existe pas les règles de formatage"), LOAD_RULES_NOT_FOUND("La norme %s n'a pas de règles de chargement"), + LOAD_RULES_SEVERAL("La norme %s a plusieurs règles de chargement"), LOAD_SEVERAL_NORM_FOUND("Plusieurs normes ou validité correspondent à l'expression : %s"), LOAD_NORM_NOT_FOUND("Aucune norme trouvée pour le fichier %s"), LOAD_ZERO_NORM_FOUND("Aucune norme trouvée"), diff --git a/arc-ws/src/main/java/fr/insee/arc/ws/services/restServices/execute/ExecuteEngineController.java b/arc-ws/src/main/java/fr/insee/arc/ws/services/restServices/execute/ExecuteEngineController.java index f834df291..74a167f28 100644 --- a/arc-ws/src/main/java/fr/insee/arc/ws/services/restServices/execute/ExecuteEngineController.java +++ b/arc-ws/src/main/java/fr/insee/arc/ws/services/restServices/execute/ExecuteEngineController.java @@ -27,6 +27,7 @@ import fr.insee.arc.core.model.TraitementPhase; import fr.insee.arc.core.service.global.bo.JeuDeRegle; import fr.insee.arc.core.service.global.bo.JeuDeRegleDao; +import fr.insee.arc.core.service.p2chargement.bo.FileIdCard; import fr.insee.arc.core.service.p2chargement.engine.ChargeurXmlComplexe; import fr.insee.arc.core.service.p3normage.engine.NormageEngine; import fr.insee.arc.core.service.p4controle.engine.ServiceJeuDeRegle; @@ -80,10 +81,13 @@ public ResponseEntity executeEngineClient( try (InputStream inputStream = new ByteArrayInputStream( bodyPojo.fileContent.getBytes(StandardCharsets.UTF_8));) { - ChargeurXmlComplexe chargeur = new ChargeurXmlComplexe(connection, bodyPojo.fileName, inputStream, currentTemporaryTable(i), - bodyPojo.norme, bodyPojo.periodicite, bodyPojo.validite, env); + + FileIdCard fileIdCard = new FileIdCard(bodyPojo.fileName); + fileIdCard.setFileAttributes(bodyPojo.norme, bodyPojo.validite, bodyPojo.periodicite); + + ChargeurXmlComplexe chargeur = new ChargeurXmlComplexe(connection, fileIdCard, inputStream, currentTemporaryTable(i)); chargeur.executeEngine(); - structure = chargeur.jointure.replace("''", "'"); + structure = chargeur.getJointure().replace("''", "'"); } break; case NORMAGE: