Merge remote-tracking branch 'origin/master'

arc-batch/src/main/java/fr/insee/arc/batch/module/MapperBatch.java

@@ -27,7 +27,7 @@ public static void main(String[] args) {
 
     @Override
     public void execute() {
-        this.report = ApiServiceFactory.getService(TypeTraitementPhase.MAPMODEL.toString(), (String) this.args[0], (String) this.args[1],
+        this.report = ApiServiceFactory.getService(TypeTraitementPhase.MAPPING.toString(), (String) this.args[0], (String) this.args[1],
                 (String) this.args[2], (String) this.args[3], (String) this.args[4]).invokeApi();
 
     }

arc-core/src/main/java/fr/insee/arc/core/factory/ApiServiceFactory.java

@@ -27,7 +27,7 @@ private ApiServiceFactory(Map<String, IServiceFactory> aMap) {
 	this.map.put(TypeTraitementPhase.STRUCTURIZE.toString(), ApiNormageServiceFactory.getInstance());
 	this.map.put(TypeTraitementPhase.CONTROL.toString(), ApiControleServiceFactory.getInstance());
 	this.map.put(TypeTraitementPhase.FILTER.toString(), ApiFiltrageServiceFactory.getInstance());
-	this.map.put(TypeTraitementPhase.MAPMODEL.toString(), ApiMappingServiceFactory.getInstance());
+	this.map.put(TypeTraitementPhase.MAPPING.toString(), ApiMappingServiceFactory.getInstance());
 	this.map.put(TypeTraitementPhase.DUMMY.toString(), ApiDummyServiceFactory.getInstance());
     }
 

arc-core/src/main/java/fr/insee/arc/core/model/TypeTraitementPhase.java

@@ -11,7 +11,7 @@ public enum TypeTraitementPhase {
     , STRUCTURIZE(4, 1000000)//
     , CONTROL(5, 1000000)//
     , FILTER(6, 1000000)//
-    , MAPMODEL(7, 1000000);//
+    , MAPPING(7, 1000000);//
 
     private TypeTraitementPhase(int anOrdre, int aNbLigneATraiter) {
 	this.order = anOrdre;

arc-core/src/main/java/fr/insee/arc/core/service/AbstractService.java

@@ -327,7 +327,7 @@ public static String temporaryTableName2(String aCurrentPhase, String tableName,
      * @param tableName
      */
     protected void addTableToBeCleandService(String tableId, String tableName) {
-	this.bddTable.addTable(tableId, this.executionEnv, tableName);
+	this.bddTable.addTemporaryTable(tableId, tableName);
 	this.idTableToClean.add(tableId);
     }
 

arc-core/src/main/java/fr/insee/arc/core/service/ApiInitialisationService.java

@@ -325,7 +325,7 @@ public void bddScript() {
 			dbEnv(executionEnv) + TypeTraitementPhase.CONTROL + "_" + TraitementState.KO,
 			dbEnv(executionEnv) + TypeTraitementPhase.FILTER + "_" + TraitementState.OK,
 			dbEnv(executionEnv) + TypeTraitementPhase.FILTER + "_" + TraitementState.KO,
-			dbEnv(executionEnv) + TypeTraitementPhase.MAPMODEL + "_" + TraitementState.KO,
+			dbEnv(executionEnv) + TypeTraitementPhase.MAPPING + "_" + TraitementState.KO,
 			dbEnv(executionEnv) + TypeTraitementPhase.LOAD + "_" + TraitementState.OK + TODO,
 			dbEnv(executionEnv) + TypeTraitementPhase.STRUCTURIZE + "_" + TraitementState.OK + TODO,
 			dbEnv(executionEnv) + TypeTraitementPhase.CONTROL + "_" + TraitementState.OK + TODO,
@@ -724,7 +724,7 @@ public void cleanPilotagetable(Connection connexion, String executionEnv) throws
 		.append(",isFichierToDelete AS (	 ").append("SELECT id_source, container, date_client ")
 		.append("FROM ").append(nomTablePilotage).append(" a ").append(", arc.ihm_norme b ")
 		.append(", clientsParFamille c ")
-		.append("WHERE a.phase_traitement='" + TypeTraitementPhase.MAPMODEL + "' ")
+		.append("WHERE a.phase_traitement='" + TypeTraitementPhase.MAPPING + "' ")
 		.append("AND a.etat_traitement='{" + TraitementState.OK + "}' ").append("AND a.client is not null ")
 		.append("AND a.id_norme=b.id_norme ").append("AND a.periodicite=b.periodicite ")
 		.append("AND b.id_famille=c.id_famille ")
@@ -1160,7 +1160,7 @@ public void synchroniserEnvironmentByPilotage(Connection connexion, String execu
 		    String etat = ManipString.substringAfterLast(nomTable, "_").toUpperCase();
 
             // TODO : add a flag to know if phase output tables are inherited tables or not
-            if (!nomTable.contains(TypeTraitementPhase.MAPMODEL.toString().toLowerCase())){
+            if (!nomTable.contains(TypeTraitementPhase.MAPPING.toString().toLowerCase())){
 
 
 		    // temporary table to store inherit table already checked

arc-core/src/main/java/fr/insee/arc/core/service/ApiReceptionService.java

@@ -323,7 +323,7 @@ public void registerFiles(Connection connexion, String anExecutionEnvironment, S
 	                	requete.append("DELETE FROM "+nomTable+" a where exists (select 1 from a_rejouer b where a.id_source=b.id_source); ");
 	                    requete.append("vacuum "+nomTable+"; ");
 
-	                    if (!nomTable.contains(TypeTraitementPhase.MAPMODEL.toString().toLowerCase())
+	                    if (!nomTable.contains(TypeTraitementPhase.MAPPING.toString().toLowerCase())
 	                            && nomTable.endsWith("_" + TraitementState.OK.toString().toLowerCase())) {
 	                    requete.append("DELETE FROM "+nomTable+"_todo a where exists (select 1 from a_rejouer b where a.id_source=b.id_source); ");
 	                    requete.append("vacuum "+nomTable+"_todo; ");	                    

arc-core/src/main/java/fr/insee/arc/core/service/mapping/RequeteMapping.java

@@ -325,7 +325,7 @@ public String getRequete(String aNomFichier) {
 
 
 		if (!this.isRequeteCalculee) {
-			StringBuilder requeteGlobale = new StringBuilder("DISCARD TEMP; SET ENABLE_HASHAGG=ON; SET ENABLE_BITMAPSCAN=OFF; \n");
+			StringBuilder requeteGlobale = new StringBuilder("SET ENABLE_HASHAGG=ON; SET ENABLE_BITMAPSCAN=OFF; \n");
 
 			construireTablePrecedente(requeteGlobale);
 

arc-core/src/main/java/fr/insee/arc/core/service/thread/AbstractThreadService.java

@@ -213,6 +213,7 @@ private void tagPilotageFinal() throws SQLException {
 	StringBuilder request = new StringBuilder();
 	request.append(this.marquageFinal(getTablePil(), this.getTablePilTempThread()));
 	request.append(FormatSQL.dropTable(this.getTablePilTempThread()));
+	request.append("\n DISCARD SEQUENCES; DISCARD TEMP;");
 	UtilitaireDao.get("arc").executeBlock(this.connection, request);
     }
 
@@ -456,6 +457,7 @@ public String marquageFinal(String tablePil, String tablePilTemp, String idSourc
 	}
 	query.append("\n \t AND a.etape = 1 ; ");
 	query.append("\n set enable_hashjoin = on; ");
+
 	return query.toString();
 
     }

arc-core/src/main/java/fr/insee/arc/core/service/thread/ThreadControleService.java

@@ -262,7 +262,7 @@ public void finControle() throws Exception {
 	blocFin.append(FormatSQL.dropTable(tableOutOkTemp).toString());
 	blocFin.append(FormatSQL.dropTable(tableOutKoTemp).toString());
 	blocFin.append(FormatSQL.dropTable(this.bddTable.getQualifedName(BddTable.ID_TABLE_POOL_DATA)).toString());
-	blocFin.append("\nDISCARD SEQUENCES; DISCARD TEMP;");
+//	blocFin.append("\nDISCARD SEQUENCES; DISCARD TEMP;");
 
 	UtilitaireDao.get("arc").executeBlock(this.connection, blocFin);
 

arc-core/src/main/java/fr/insee/arc/core/service/thread/ThreadIdentifyService.java

@@ -73,13 +73,13 @@ public ThreadIdentifyService(int currentIndice, ApiIdentifyService aApi, Connect
      * 
      * @throws SQLException
      */
-    private void cleanBdd() throws SQLException {
-	// Nettoyage
-	StringBuilder blocFin = new StringBuilder();
-
-	blocFin.append("\nDISCARD SEQUENCES; DISCARD TEMP;");
-	UtilitaireDao.get("arc").executeBlock(this.connection, blocFin);
-    }
+//    private void cleanBdd() throws SQLException {
+//	// Nettoyage
+//	StringBuilder blocFin = new StringBuilder();
+//
+//	blocFin.append("\nDISCARD SEQUENCES; DISCARD TEMP;");
+//	UtilitaireDao.get("arc").executeBlock(this.connection, blocFin);
+//    }
 
     /**
      * Traitement métier lié à l'identification.
@@ -215,7 +215,7 @@ public void process() throws Exception {
 
     @Override
     public void finalizePhase() throws SQLException {
-	cleanBdd();
+//	cleanBdd();
 
     }
 

arc-core/src/main/java/fr/insee/arc/core/service/thread/ThreadLoadService.java

@@ -138,7 +138,7 @@ public void finalizePhase() throws Exception {
 	StringBuilder blocFin = new StringBuilder();
 	blocFin.append(FormatSQL.dropTable(this.tableTempA));
 	blocFin.append(FormatSQL.dropTable(this.tableTempAll));
-	blocFin.append("\nDISCARD SEQUENCES; DISCARD TEMP;");
+//	blocFin.append("\nDISCARD SEQUENCES; DISCARD TEMP;");
 	UtilitaireDao.get("arc").executeBlock(this.connection, blocFin);
 
     }

arc-core/src/main/java/fr/insee/arc/core/service/thread/ThreadStructurizeService.java

@@ -27,6 +27,7 @@
 import fr.insee.arc.core.model.rules.RuleStructurizeEntity;
 import fr.insee.arc.core.service.ApiNormageService;
 import fr.insee.arc.utils.dao.UtilitaireDao;
+import fr.insee.arc.utils.format.Format;
 import fr.insee.arc.utils.structure.GenericBean;
 import fr.insee.arc.utils.utils.FormatSQL;
 import fr.insee.arc.utils.utils.LoggerDispatcher;
@@ -191,6 +192,8 @@ private void joinXMLBlocks() throws Exception {
 	} else {
 	    jointure = jointure.toLowerCase();
 
+	    if (Format.size(regle.get("id_regle"))>0)
+	    {
 	    // ORDRE IMPORTANT
 	    // on supprime les rubriques inutilisées quand le service est invoqué en batch
 	    // (this.paramBatch!=null)
@@ -213,7 +216,9 @@ private void joinXMLBlocks() throws Exception {
 	    jointure = appliquerRegleUnicite(regle, norme, validite, periodicite, jointure);
 
 	    jointure = appliquerRegleRelation(regle, norme, validite, periodicite, jointure);
-
+
+	    }
+
 	    // optimisation manu 9.6
 	    // retravaille de la requete pour éliminer UNION ALL
 	    jointure = optimisation96(jointure);

arc-core/src/main/resources/BdD/script_table.sql

@@ -39,6 +39,7 @@ EXCEPTION WHEN OTHERS then end; $$;
 update arc.ihm_paramettrage_ordre_phase set nom_phase='STRUCTURIZE', type_phase='STRUCTURIZE' where nom_phase='STRUCTURIZE_XML';
 update arc.ihm_paramettrage_ordre_phase set phase_precedente='STRUCTURIZE' where phase_precedente='STRUCTURIZE_XML';
 update arc.ihm_paramettrage_ordre_phase set nom_phase='MAPMODEL', type_phase='MAPMODEL' where nom_phase='FORMAT_TO_MODEL';
+update arc.ihm_paramettrage_ordre_phase set nom_phase='MAPPING', type_phase='MAPPING' where nom_phase='MAPMODEL';
 
 
 CREATE TABLE IF NOT EXISTS arc.ext_etat (
@@ -71,6 +72,7 @@ exception
   when others then
 end;
 $$;
+UPDATE arc.ext_etat_jeuderegle set id='arc.bas1', val='BAC A SABLE 1' where id='arc.bas';
 CREATE TABLE IF NOT EXISTS arc.ext_etat_jeuderegle (
   id text NOT NULL,
   val text,
@@ -428,6 +430,10 @@ CREATE TABLE IF NOT EXISTS arc.ihm_mod_variable_metier (
   ),
   CONSTRAINT fk_ihm_mod_variable_table_metier FOREIGN KEY (id_famille, nom_table_metier) REFERENCES arc.ihm_mod_table_metier (id_famille, nom_table_metier) MATCH SIMPLE ON UPDATE NO ACTION ON DELETE NO ACTION
 );
+
+alter table arc.ihm_mod_variable_metier alter column type_consolidation set default '{exclus}';
+
+
 CREATE TABLE IF NOT EXISTS arc.ihm_nmcl (
   nom_table text NOT NULL,
   description text,

arc-utils/src/main/java/fr/insee/arc/utils/format/Format.java

@@ -30,6 +30,11 @@ public static final String removeJSONForbiddenChars(String string) {
         return string.replaceAll("[\r\n\t]", " ");
     }
 
+    public static <T> int size(ArrayList<T> o)
+    {
+		return o==null?0:o.size();
+    }
+
     /**
      *
      * @param list

arc-web/src/main/java/fr/insee/arc/web/action/GererFamilleNormeAction.java

@@ -15,6 +15,7 @@
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.stereotype.Component;
 
+import fr.insee.arc.core.model.TypeTraitementPhase;
 import fr.insee.arc.utils.dao.UtilitaireDao;
 import fr.insee.arc.utils.utils.FormatSQL;
 import fr.insee.arc.utils.utils.LoggerDispatcher;
@@ -279,7 +280,7 @@ public static StringBuilder getRequeteListeVariableMetierTableMetier(List<String
         StringBuilder left = new StringBuilder("(SELECT nom_variable_metier");
         for (int i = 0; i < listeTableMetier.size(); i++) {
             left.append(",\n  CASE WHEN '['||string_agg(nom_table_metier,'][' ORDER BY nom_table_metier)||']' LIKE '%['||'" + listeTableMetier.get(i)
-                    + "'||']%' then 'oui' else '' end " + listeTableMetier.get(i));
+                    + "'||']%' then 'x' else '' end " + listeTableMetier.get(i));
         }
         left.append("\nFROM arc."+IHM_MOD_VARIABLE_METIER+" WHERE id_famille='" + idFamille + "'\n");
         left.append("GROUP BY nom_variable_metier) left_side");
@@ -447,7 +448,9 @@ private String addNonExistingVariableMetierWithoutSync(StringBuilder message) {
             StringBuilder requete = new StringBuilder();
             boolean blank=true;
             for (int i = numberOfColumnTableVariableMetier; i < this.viewVariableMetier.getInputFields().size(); i++) {
-                if (StringUtils.isNotBlank(this.viewVariableMetier.getInputFields().get(i)) && this.viewVariableMetier.getInputFields().get(i).equals("oui")) {
+                if (StringUtils.isNotBlank(this.viewVariableMetier.getInputFields().get(i))
+                		// && this.viewVariableMetier.getInputFields().get(i).equals("oui")
+                		) {
 
                 	// au moins une table est resnseignée
                 	blank=false;
@@ -690,7 +693,7 @@ private String deleteVariableMetierWithoutSync(StringBuilder message, Map<String
 
     private boolean isMofificationOk(StringBuilder message, HashMap<String, ArrayList<String>> mapContentAfterUpdate) {
         return estCeQueLesNomsDeVariablesSontNonNuls(message, mapContentAfterUpdate)
-                && estCeQueLesIdentifiantsSontExclus(message, mapContentAfterUpdate)
+               // && estCeQueLesIdentifiantsSontExclus(message, mapContentAfterUpdate)
                 && estCeQueLeSchemaNeComportePasDeCycles(message, mapContentAfterUpdate);
     }
 
@@ -794,7 +797,7 @@ public static final void executeRequeteMiseAjourTableMetier(StringBuilder messag
     }
 
     public final boolean isNomTableMetierValide(String nomTable) {
-        return nomTable.matches("(?i)^mapping_" + this.viewFamilleNorme.mapContentSelected().get("id_famille").get(0) + "_([a-z]|_)*[a-z]+_ok$");
+        return nomTable.matches("(?i)^"+TypeTraitementPhase.MAPPING.toString().toLowerCase()+"_" + this.viewFamilleNorme.mapContentSelected().get("id_famille").get(0) + "_([a-z]|_)*[a-z]+_ok$");
     }
 
     @Action(value = "/addTableMetier")

arc-web/src/main/java/fr/insee/arc/web/action/GererNormeAction.java

@@ -955,8 +955,8 @@ public String preGenererRegleFiltrage() {
 	    UtilitaireDao.get("arc").executeRequest(null,
 		    new StringBuilder("INSERT INTO " + this.viewFiltrage.getTable())//
 			    .append("  " + Format.stringListe(this.viewFiltrage.getDatabaseColumnsLabel()))//
-			    .append("  SELECT (SELECT max(id_regle) FROM " + this.viewFiltrage.getTable()
-				    + ")+row_number() over () ,")//
+			    .append("  SELECT coalesce((SELECT max(id_regle) FROM " + this.viewFiltrage.getTable()
+				    + "),0)+row_number() over () ,")//
 			    .append("  '"
 				    + this.viewRulesSet.mapContentSelected().get(ConstanteBD.ID_NORME.getValue()).get(0)
 				    + "', ")//
@@ -1065,8 +1065,8 @@ public String preGenererRegleMapping() {
 	    // List hard coded to be sure of the order in the select
 	    StringBuilder requete = new StringBuilder("INSERT INTO " + this.viewMapping.getTable()).append(
 		    "  (id_regle, id_norme, validite_inf, validite_sup,  version , periodicite, variable_sortie, expr_regle_col, commentaire) ")
-		    .append("  SELECT (SELECT max(id_regle) FROM " + this.viewMapping.getTable()
-			    + ")+row_number() over () ,")
+		    .append("  SELECT coalesce((SELECT max(id_regle) FROM " + this.viewMapping.getTable()
+			    + "),0)+row_number() over () ,")
 		    .append("  '" + this.viewRulesSet.mapContentSelected().get(ConstanteBD.ID_NORME.getValue()).get(0)
 			    + "', ")
 		    .append("  '"

arc-web/src/main/java/fr/insee/arc/web/action/PilotageBAS8Action.java

@@ -169,9 +169,11 @@ public String sortRapportBAS8() {
 
 	@Action(value = "/filesUploadBAS8")
 	public String filesUploadBAS8() {
-		initialize();
 		LoggerHelper.debug(LOGGER, "* /* filesUploadBAS : " + this.viewEntrepotBAS8.getCustomValues() + " */ *");
 
+		initializeArcActionWithProperties();
+		recupererEnvironnementTravail();
+
 		if (this.viewEntrepotBAS8.getCustomValues() != null
 				&& !this.viewEntrepotBAS8.getCustomValues().get(WRITING_REPO).equals("")
 				&& this.viewPilotageBAS8.getFileUploadFileName() != null) {
@@ -202,6 +204,9 @@ public String filesUploadBAS8() {
 		ApiServiceFactory.getService(TypeTraitementPhase.REGISTER.toString(), "arc.ihm",
 				(String) getSession().get(SessionParameters.ENV), this.repertoire,
 				String.valueOf(TypeTraitementPhase.REGISTER.getNbLinesToProcess())).invokeApi();
+
+		initialize();
+
 		return generateDisplay();
 
 	}

arc-web/src/main/java/fr/insee/arc/web/action/PilotagePRODAction.java

@@ -475,8 +475,8 @@ public String startMappingPROD() {
         ApiInitialisationService.copyTablesToExecution(null, "arc.ihm", this.envExecution);
 
         // String repertoire=ServletActionContext.getServletContext().getRealPath("/");
-        ApiServiceFactory.getService(TypeTraitementPhase.MAPMODEL.toString(), "arc.ihm", this.envExecution, this.repertoire,
-                String.valueOf(TypeTraitementPhase.MAPMODEL.getNbLinesToProcess()),new SimpleDateFormat("yyyyMMddHH").format(new Date())).invokeApi();
+        ApiServiceFactory.getService(TypeTraitementPhase.MAPPING.toString(), "arc.ihm", this.envExecution, this.repertoire,
+                String.valueOf(TypeTraitementPhase.MAPPING.getNbLinesToProcess()),new SimpleDateFormat("yyyyMMddHH").format(new Date())).invokeApi();
         return sessionSyncronize();
     }
 

arc-web/src/main/java/fr/insee/arc/web/model/ViewVariableMetier.java

@@ -37,7 +37,7 @@ public static HashMap<String, ColumnRendering> getInitialRenderingViewVariableMe
         returned.put("description_variable_metier", new ColumnRendering(true, "label.comment", "200px", "text", null, true));
         returned.put("type_variable_metier", new ColumnRendering(true, "label.mapmodel.field.type", "100px", "select",
                 "SELECT nom_type id, nom_type val FROM arc.ext_mod_type_autorise ORDER BY nom_type", true));
-        returned.put("type_consolidation", new ColumnRendering(true, "label.mapmodel.field.aggregate", "200px", "text", null, true));
+        returned.put("type_consolidation", new ColumnRendering(false, "label.mapmodel.field.aggregate", "200px", "text", null, true));
         return returned;
     }
 

arc-web/src/main/resources/global.properties

@@ -104,7 +104,7 @@ view.load=Load rules
 view.structurize=Structurize rules
 view.control=Control rules
 view.filter=Filter rules
-view.mapping=Map to model rules
+view.mapping=Mapping rules
 
 view.envManagement=Files status in the workflow
 view.envManagement.report=Workflow messages report

arc-web/src/main/resources/global_en.properties

@@ -103,7 +103,7 @@ view.load=Load rules
 view.structurize=Structurize rules
 view.control=Control rules
 view.filter=Filter rules
-view.mapping=Map to model rules
+view.mapping=Mapping rules
 
 view.envManagement=Files status in the workflow
 view.envManagement.report=Workflow messages report

arc-web/src/main/webapp/js/component.js

@@ -1,5 +1,4 @@
-
-//Modules disponibles
+//Modules disponibles
 //Render:Tree;
 //Render:Pilotage;
 //Render:AlertBox;
@@ -756,7 +755,7 @@ function updateCells(e,t)
 	var view=$(t).parents().filter('div[id]').attr('id');
 
 	// validation avec entrée
-	if (e.keyCode == 13) {
+	if (e.ctrlKey && e.keyCode == 13) {
 		e.preventDefault();
 		$('[id="'+view+'.update"]').trigger('click');
 	}

arc-web/src/main/webapp/jsp/gererNomenclature.jsp

@@ -92,6 +92,9 @@
 							</s:include>
 						</div>
 					</div>
+
+					 <s:file name="fileUpload" label="Select a File to upload" size="40" />
+					 <input type="submit" id="btnFileUpload" value=<s:property value="getText('managementSandbox.load')"/> scope="" doAction="importListNomenclatures" onclick="submitForm()" style="margin-left: 25px;" ajax="false" />
 			</div>
 
 				<div class="col-md-2 border-right" style="margin-top: 2.25rem;"

arc-web/src/main/webapp/jsp/gererNorme.jsp

@@ -65,7 +65,7 @@
 
 			<div class="row">
 				<!-- left column -->
-				<div class="col-md-5 border-right"
+				<div class="col-md-6 border-right"
 				>
 					<div class="row">
 						<div class="col-md">
@@ -152,7 +152,7 @@
 
 				</div>
 
-				<div class="col-md-7">
+				<div class="col-md-6">
 					<div class="row">
 						<div  id="viewModuleButtons" class="col-md">
 							<!-- tab to choose the module -->