diff --git a/arc-utils/src/main/java/fr/insee/arc/utils/dao/UtilitaireDao.java b/arc-utils/src/main/java/fr/insee/arc/utils/dao/UtilitaireDao.java index 6d6a76352..1626b74cf 100644 --- a/arc-utils/src/main/java/fr/insee/arc/utils/dao/UtilitaireDao.java +++ b/arc-utils/src/main/java/fr/insee/arc/utils/dao/UtilitaireDao.java @@ -53,7 +53,6 @@ public class UtilitaireDao implements IConstanteNumerique, IConstanteCaractere { private static final Logger LOGGER = LogManager.getLogger(UtilitaireDao.class); - /** * execute request returns a table with headers, type and data provide the * indexes of these elements @@ -90,19 +89,21 @@ public static final UtilitaireDao get(Integer aPool) { public int numberOfNods() { return properties.getConnectionProperties().size(); } - - /** return a valid connection index according to the given connection in properties + + /** + * return a valid connection index according to the given connection in + * properties + * * @param aPool * @return */ private int validConnectionIndex(Integer aPool) { - + int numberOfNodsDeclaredInProperties = numberOfNods(); - - return (aPool someTables) { ex.logFullException(); } } - /** * Exécute une requête qui renvoie exactement UN (unique) résultat de type @@ -250,9 +251,9 @@ public int getInt(Connection connexion, GenericPreparedStatementBuilder sql, Mod return ZERO; } - /** * Check if a column exists in a table + * * @param aConnexion * @param aNomTable * @param aNomVariable @@ -285,7 +286,7 @@ public void executeImmediate(Connection connexion, StringBuilder requete, ModeRe throws ArcException { executeImmediate(connexion, requete.toString(), modes); } - + public void executeImmediate(Connection connexion, GenericPreparedStatementBuilder requete, ModeRequete... modes) throws ArcException { executeImmediate(connexion, requete.getQueryWithParameters(), modes); @@ -369,7 +370,6 @@ public List> executeRequest(Connection connexion, GenericPreparedSt } - /** * Register in the targetPreparedStatement the bind variable of a request with * correct type @@ -566,7 +566,6 @@ public void outStreamRequeteSelect(Connection connexion, GenericPreparedStatemen } } - /** * Les fichiers à copier sont potentiellement dans des dossiers différents * (****_OK ou *****_KO)
@@ -616,7 +615,8 @@ public void getFilesDataStreamFromListOfInputDirectories(Connection connexion, taos.putArchiveEntry(entry); // Ecriture dans le fichier CompressedUtils.copyFromInputstreamToOutputStream( - new BufferedInputStream(new FileInputStream(fileIn), CompressedUtils.READ_BUFFER_SIZE), taos); + new BufferedInputStream(new FileInputStream(fileIn), CompressedUtils.READ_BUFFER_SIZE), + taos); taos.closeArchiveEntry(); } } @@ -689,7 +689,6 @@ public List getList(Connection connexion, String requete, List r return returned; } - /** * Postgres libère mal l'espace sur les tables quand on fait trop d'opération * sur les colonnes. Un vacuum full des tables du méta-modèle permet de résoudre @@ -733,8 +732,7 @@ public void maintenancePgCatalog(Connection connexion, String type) { */ public Collection getColumns(Connection connexion, Collection liste, String tableIn) throws ArcException { - liste.addAll( - new GenericBean(executeRequest(connexion, FormatSQL.listeColonneByHeaders(tableIn))).getHeaders()); + liste.addAll(new GenericBean(executeRequest(connexion, FormatSQL.listeColonneByHeaders(tableIn))).getHeaders()); return liste; } @@ -757,8 +755,8 @@ public List getColumns(Connection connexion, String tableIn) throws ArcE * jour "12 as a" * @throws ArcException */ - public void fastUpdate(Connection aConnexion, String tableName, String keys, - List colList, String where, String... set) throws ArcException { + public void fastUpdate(Connection aConnexion, String tableName, String keys, List colList, String where, + String... set) throws ArcException { // récupérer la liste des colonnes // liste de toutes les colonnes // liste des colonnes à mettre à jour @@ -864,30 +862,14 @@ public void fastUpdate(Connection aConnexion, String tableName, String keys, * @throws ArcException * @throws IOException */ - public void exporting(Connection connexion, String table, OutputStream os, boolean csv, boolean... forceQuote) - throws ArcException { + public void exporting(Connection connexion, String table, OutputStream os, boolean csv) throws ArcException { ConnectionWrapper conn = initConnection(connexion); - boolean forceQuoteBis; - if (forceQuote != null && forceQuote.length > 0) { - forceQuoteBis = forceQuote[0]; - } else { - forceQuoteBis = true; - } - try { CopyManager copyManager = new CopyManager((BaseConnection) conn.getConnexion()); if (csv) { - if (forceQuoteBis) { - copyManager.copyOut("COPY " + table - + " TO STDOUT WITH (FORMAT csv, HEADER true , DELIMITER ';' , FORCE_QUOTE *, ENCODING 'UTF8') ", - os); - } else { - copyManager.copyOut( - "COPY " + table - + " TO STDOUT WITH (FORMAT csv, HEADER true , DELIMITER ';' , ENCODING 'UTF8') ", - os); - } + copyManager.copyOut("COPY " + table + + " TO STDOUT WITH (FORMAT csv, HEADER false , DELIMITER ';' , ENCODING 'UTF8') ", os); } else { copyManager.copyOut("COPY " + table + " TO STDOUT WITH (FORMAT BINARY)", os); } diff --git a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ImportStep1InitializeClientTablesService.java b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ImportStep1InitializeClientTablesService.java index 26b14b36e..d851e5262 100644 --- a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ImportStep1InitializeClientTablesService.java +++ b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ImportStep1InitializeClientTablesService.java @@ -14,6 +14,7 @@ import fr.insee.arc.utils.exception.ArcExceptionMessage; import fr.insee.arc.ws.services.importServlet.actions.SendResponse; import fr.insee.arc.ws.services.importServlet.bo.ArcClientIdentifier; +import fr.insee.arc.ws.services.importServlet.bo.ExportSource; import fr.insee.arc.ws.services.importServlet.bo.JsonKeys; import fr.insee.arc.ws.services.importServlet.dao.ClientDao; @@ -45,8 +46,8 @@ public ImportStep1InitializeClientTablesService(JSONObject dsnRequest) { private List sources; - private void executeIf(String source, Executable exe) throws ArcException { - if (!sources.contains(source)) { + private void executeIf(ExportSource source, Executable exe) throws ArcException { + if (!sources.contains(source.getSource())) { return; } exe.execute(); @@ -93,13 +94,13 @@ private void startTableCreationInParallel() { public void run() { try { if (tablesMetierNames != null) { - executeIf(ServletArc.MAPPING, () -> clientDao.createImages(tablesMetierNames)); - executeIf(ServletArc.METADATA, () -> clientDao.createTableMetier()); - executeIf(ServletArc.METADATA, () -> clientDao.createVarMetier()); + executeIf(ExportSource.MAPPING, () -> clientDao.createImages(tablesMetierNames)); + executeIf(ExportSource.METADATA, () -> clientDao.createTableMetier()); + executeIf(ExportSource.METADATA, () -> clientDao.createVarMetier()); } - executeIf(ServletArc.NOMENCLATURE, () -> clientDao.createNmcl()); - executeIf(ServletArc.METADATA, () -> clientDao.createTableFamille()); - executeIf(ServletArc.METADATA, () -> clientDao.createTablePeriodicite()); + executeIf(ExportSource.NOMENCLATURE, () -> clientDao.createNmcl()); + executeIf(ExportSource.METADATA, () -> clientDao.createTableFamille()); + executeIf(ExportSource.METADATA, () -> clientDao.createTablePeriodicite()); } catch (ArcException e) { try { clientDao.createTableWsKO(); diff --git a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ImportStep3GetTableDataService.java b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ImportStep3GetTableDataService.java index 1e76e23cf..b6a004721 100644 --- a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ImportStep3GetTableDataService.java +++ b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ImportStep3GetTableDataService.java @@ -9,6 +9,7 @@ import fr.insee.arc.utils.utils.Sleep; import fr.insee.arc.ws.services.importServlet.actions.SendResponse; import fr.insee.arc.ws.services.importServlet.bo.ArcClientIdentifier; +import fr.insee.arc.ws.services.importServlet.bo.ExportFormat; import fr.insee.arc.ws.services.importServlet.dao.ClientDao; import fr.insee.arc.ws.services.importServlet.dao.ServiceDao; @@ -16,7 +17,6 @@ public class ImportStep3GetTableDataService { protected static final Logger LOGGER = LogManager.getLogger(ImportStep3GetTableDataService.class); - private static final boolean IS_EXPORT_CSV = false; // delay to wait before requiring a new table when webservice is still creating tables to retrieve private static final int WAIT_DELAY_ON_PENDING_TABLES_CREATION_IN_MS = 10000; @@ -37,7 +37,7 @@ public void execute(SendResponse resp) throws ArcException { // binary transfer ServiceDao.execQueryExportDataToResponse(resp.getWr(), - ViewEnum.normalizeTableName(arcClientIdentifier.getClient()), IS_EXPORT_CSV); + ViewEnum.normalizeTableName(arcClientIdentifier.getClient()), this.arcClientIdentifier.getFormat().equals(ExportFormat.CSV_GZIP.getFormat())); if (this.clientDao.isWebServiceNotPending()) { this.clientDao.dropTable(arcClientIdentifier.getClient()); diff --git a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ServletArc.java b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ServletArc.java index b2a29e3f6..85100d8f1 100644 --- a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ServletArc.java +++ b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ServletArc.java @@ -24,6 +24,8 @@ import fr.insee.arc.utils.webutils.WebUtils; import fr.insee.arc.ws.services.importServlet.actions.InitiateRequest; import fr.insee.arc.ws.services.importServlet.actions.SendResponse; +import fr.insee.arc.ws.services.importServlet.bo.ExportFormat; +import fr.insee.arc.ws.services.importServlet.bo.ExportSource; import fr.insee.arc.ws.services.importServlet.bo.JsonKeys; import fr.insee.arc.ws.services.importServlet.dao.SecurityDao; @@ -31,11 +33,7 @@ public class ServletArc extends HttpServlet { private static final long serialVersionUID = 1L; - public static final String NOMENCLATURE = "nomenclature"; - public static final String METADATA = "metadata"; - public static final String MAPPING = "mapping"; - - private static final List DEFAULT_SOURCE = Arrays.asList(MAPPING, NOMENCLATURE, METADATA); + private static final List DEFAULT_SOURCE = Arrays.asList(ExportSource.MAPPING.getSource(), ExportSource.NOMENCLATURE.getSource(), ExportSource.METADATA.getSource()); private static final Logger LOGGER = LogManager.getLogger(ServletArc.class); @@ -103,7 +101,11 @@ private JSONObject buildRequest(HttpServletRequest request) { // get parameters from request JSONObject returned = new JSONObject(request.getParameter("requests")); - + + if (returned.isNull(JsonKeys.FORMAT.getKey())) { + returned.put(JsonKeys.FORMAT.getKey(), ExportFormat.BINARY.getFormat()); + } + // if SOURCE key is not specified, add all the default sources to be retrieved if (returned.isNull(JsonKeys.SOURCE.getKey())) { return returned.put(JsonKeys.SOURCE.getKey(), DEFAULT_SOURCE); diff --git a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/actions/SendResponse.java b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/actions/SendResponse.java index 34c659316..03a11026f 100644 --- a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/actions/SendResponse.java +++ b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/actions/SendResponse.java @@ -12,73 +12,15 @@ import fr.insee.arc.utils.exception.ArcException; import fr.insee.arc.utils.utils.LoggerHelper; - - -/**Cette classe permet d'envoyer la réponse au client via un flux compressé. - * La réponse aura une des formes suivantes : - * - * Dans le cas d'un appel au service "arcClient" : - * { - * "type":"jsonwsp/response", - * "responses": - * [ - * { - * "id":"int", - * "table": - * [ - * "nomColonne1;nomColonne2;...;nomColonnen;", - * "typeColonne1;typeColonne2;...;typeColonnen;", - * "a11;a12;...;a1n;", - * "...", - * "an1;an2;...;ann"; - * ] - * }, - * ], - * "nomenclatures" : - * [ - * { - * "id":"int", - * "table": - * [ - * "nomColonne1;nomColonne2;...;nomColonnen;", - * "typeColonne1;typeColonne2;...;typeColonnen;", - * "a11;a12;...;a1n;", - * "...", - * "an1;an2;...;ann;" - * ] - * }, - * ], - * "varMetier" : - * [ - * { - * "id":"int", - * "table": - * [ - * "nomColonne1;nomColonne2;...;nomColonnen;", - * "typeColonne1;typeColonne2;...;typeColonnen;", - * "a11;a12;...;a1n;", - * "...", - * "an1;an2;...;ann;" - * ] - * }, - * ] - * } - * - * - dans le cas d'une erreur : - * { - * "type":"jsonwsp/response", - * "error":"string" - * } - * - * @author N6YF91 - * +/** + * Cette classe permet d'envoyer la réponse au client */ public class SendResponse { private static final Logger LOGGER = LogManager.getLogger(SendResponse.class); private ServletOutputStream wr; - private HttpServletResponse response; + private HttpServletResponse response; public SendResponse( HttpServletResponse response ){ this.response = response; @@ -125,19 +67,8 @@ public void endSending(){ } } - - public ServletOutputStream getWr() { return wr; } - - - - public void setWr(ServletOutputStream wr) { - this.wr = wr; - } - - - } diff --git a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/ArcClientIdentifier.java b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/ArcClientIdentifier.java index 7a41c6f92..a5e4bb44f 100644 --- a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/ArcClientIdentifier.java +++ b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/ArcClientIdentifier.java @@ -12,12 +12,11 @@ public class ArcClientIdentifier { public ArcClientIdentifier(JSONObject dsnRequest) { this.dsnRequest = dsnRequest; - this.client = getKeyIfExists(JsonKeys.CLIENT); this.timestamp = System.currentTimeMillis(); this.environnement = getKeyIfExists(JsonKeys.ENVIRONNEMENT, Patch::normalizeSchemaName); this.famille = getKeyIfExists(JsonKeys.FAMILLE); - + this.format = getKeyIfExists(JsonKeys.FORMAT); } private JSONObject dsnRequest; @@ -30,6 +29,8 @@ public ArcClientIdentifier(JSONObject dsnRequest) { private String famille; + private String format; + private String getKeyIfExists(JsonKeys key, UnaryOperator f ) { @@ -57,6 +58,9 @@ public String getClient() { public String getFamille() { return famille; } - + + public String getFormat() { + return format; + } } diff --git a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/ExportFormat.java b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/ExportFormat.java new file mode 100644 index 000000000..cda9690cc --- /dev/null +++ b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/ExportFormat.java @@ -0,0 +1,18 @@ +package fr.insee.arc.ws.services.importServlet.bo; + +public enum ExportFormat { + + CSV_GZIP("csv_gzip"), BINARY("binary"); + + private String format; + + private ExportFormat(String format) + { + this.format = format; + } + + public String getFormat() { + return format; + } + +} diff --git a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/ExportSource.java b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/ExportSource.java new file mode 100644 index 000000000..fa88ef31a --- /dev/null +++ b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/ExportSource.java @@ -0,0 +1,17 @@ +package fr.insee.arc.ws.services.importServlet.bo; + +public enum ExportSource { + + NOMENCLATURE("nomenclature"), METADATA("metadata"), MAPPING("mapping"); + + private String source; + + private ExportSource(String source) { + this.source = source; + } + + public String getSource() { + return source; + } + +} diff --git a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/JsonKeys.java b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/JsonKeys.java index 7202d5e5a..1a867f26c 100644 --- a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/JsonKeys.java +++ b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/JsonKeys.java @@ -20,6 +20,10 @@ public enum JsonKeys { ,FILEDATA("filedata") ,SOURCE("source") + // + ,FORMAT("format") + ,ACCESS_TOKEN("acessToken") + ; private String key; diff --git a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/dao/ServiceDao.java b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/dao/ServiceDao.java index 6ec357c4f..41d45b3b0 100644 --- a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/dao/ServiceDao.java +++ b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/dao/ServiceDao.java @@ -1,15 +1,30 @@ package fr.insee.arc.ws.services.importServlet.dao; +import java.io.IOException; import java.io.OutputStream; +import java.util.zip.GZIPOutputStream; import fr.insee.arc.utils.dao.UtilitaireDao; import fr.insee.arc.utils.exception.ArcException; +import fr.insee.arc.utils.exception.ArcExceptionMessage; public class ServiceDao { - public static void execQueryExportDataToResponse(OutputStream os, String tableName, boolean csvExportFormat) throws ArcException { - UtilitaireDao.get(0).exporting(null, tableName, os, csvExportFormat); + + if (csvExportFormat) + { + try(GZIPOutputStream goz=new GZIPOutputStream(os);) + { + UtilitaireDao.get(0).exporting(null, tableName, goz, csvExportFormat); + } catch (IOException e) { + throw new ArcException(ArcExceptionMessage.STREAM_WRITE_FAILED); + } + } + else + { + UtilitaireDao.get(0).exporting(null, tableName, os, csvExportFormat); + } } }