Skip to content

Commit

Permalink
feat: data retrieval webservice output with format = csv_gzip or binary
Browse files Browse the repository at this point in the history
  • Loading branch information
@Nolife999
Nolife999 committed Nov 24, 2023
1 parent 4513bf3 commit b90fe54
Show file tree
Hide file tree
Showing 10 changed files with 114 additions and 140 deletions.
76 changes: 29 additions & 47 deletions arc-utils/src/main/java/fr/insee/arc/utils/dao/UtilitaireDao.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@ public class UtilitaireDao implements IConstanteNumerique, IConstanteCaractere {

private static final Logger LOGGER = LogManager.getLogger(UtilitaireDao.class);


/**
* execute request returns a table with headers, type and data provide the
* indexes of these elements
Expand Down Expand Up @@ -90,19 +89,21 @@ public static final UtilitaireDao get(Integer aPool) {
public int numberOfNods() {
return properties.getConnectionProperties().size();
}

/** return a valid connection index according to the given connection in properties

/**
* return a valid connection index according to the given connection in
* properties
*
* @param aPool
* @return
*/
private int validConnectionIndex(Integer aPool) {

int numberOfNodsDeclaredInProperties = numberOfNods();
return (aPool<numberOfNodsDeclaredInProperties)?aPool:numberOfNodsDeclaredInProperties-1;

return (aPool < numberOfNodsDeclaredInProperties) ? aPool : numberOfNodsDeclaredInProperties - 1;
}



/**
* Retourne une connexion vers la base de données
*
Expand All @@ -113,10 +114,11 @@ private int validConnectionIndex(Integer aPool) {
public final Connection getDriverConnexion() throws ArcException {
// invocation du driver
try {

int validConnectionIndex=validConnectionIndex(this.pool);

ConnectionAttribute currentConnectionAttributes = properties.getConnectionProperties().get(validConnectionIndex);

int validConnectionIndex = validConnectionIndex(this.pool);

ConnectionAttribute currentConnectionAttributes = properties.getConnectionProperties()
.get(validConnectionIndex);
String driver = currentConnectionAttributes.getDatabaseDriverClassName();
String uri = currentConnectionAttributes.getDatabaseUrl();
String user = currentConnectionAttributes.getDatabaseUsername();
Expand All @@ -131,12 +133,12 @@ public final Connection getDriverConnexion() throws ArcException {
props.setProperty("user", user);
props.setProperty("password", password);
props.setProperty("tcpKeepAlive", "true");
props.setProperty("prepareThreshold" , "0");
props.setProperty("preparedStatementCacheQueries" , "0");
props.setProperty("preparedStatementCacheSizeMiB" , "0");
props.setProperty("prepareThreshold", "0");
props.setProperty("preparedStatementCacheQueries", "0");
props.setProperty("preparedStatementCacheSizeMiB", "0");

c = DriverManager.getConnection(uri, props);

} catch (Exception e) {
throw new ArcException(e, ArcExceptionMessage.DATABASE_CONNECTION_FAILED);
}
Expand Down Expand Up @@ -195,7 +197,7 @@ public void dropTable(Connection connexion, String... someTables) {
ex.logFullException();
}
}

/**
* @param connexion la connexion à la base
* @param someTables le nom des tables
Expand All @@ -208,7 +210,6 @@ public void dropTable(Connection connexion, List<String> someTables) {
ex.logFullException();
}
}


/**
* Exécute une requête qui renvoie exactement UN (unique) résultat de type
Expand Down Expand Up @@ -250,9 +251,9 @@ public int getInt(Connection connexion, GenericPreparedStatementBuilder sql, Mod
return ZERO;
}


/**
* Check if a column exists in a table
*
* @param aConnexion
* @param aNomTable
* @param aNomVariable
Expand Down Expand Up @@ -285,7 +286,7 @@ public void executeImmediate(Connection connexion, StringBuilder requete, ModeRe
throws ArcException {
executeImmediate(connexion, requete.toString(), modes);
}

public void executeImmediate(Connection connexion, GenericPreparedStatementBuilder requete, ModeRequete... modes)
throws ArcException {
executeImmediate(connexion, requete.getQueryWithParameters(), modes);
Expand Down Expand Up @@ -369,7 +370,6 @@ public List<List<String>> executeRequest(Connection connexion, GenericPreparedSt

}


/**
* Register in the targetPreparedStatement the bind variable of a request with
* correct type
Expand Down Expand Up @@ -566,7 +566,6 @@ public void outStreamRequeteSelect(Connection connexion, GenericPreparedStatemen
}
}


/**
* Les fichiers à copier sont potentiellement dans des dossiers différents
* (****_OK ou *****_KO) <br/>
Expand Down Expand Up @@ -616,7 +615,8 @@ public void getFilesDataStreamFromListOfInputDirectories(Connection connexion,
taos.putArchiveEntry(entry);
// Ecriture dans le fichier
CompressedUtils.copyFromInputstreamToOutputStream(
new BufferedInputStream(new FileInputStream(fileIn), CompressedUtils.READ_BUFFER_SIZE), taos);
new BufferedInputStream(new FileInputStream(fileIn), CompressedUtils.READ_BUFFER_SIZE),
taos);
taos.closeArchiveEntry();
}
}
Expand Down Expand Up @@ -689,7 +689,6 @@ public List<String> getList(Connection connexion, String requete, List<String> r
return returned;
}


/**
* Postgres libère mal l'espace sur les tables quand on fait trop d'opération
* sur les colonnes. Un vacuum full des tables du méta-modèle permet de résoudre
Expand Down Expand Up @@ -733,8 +732,7 @@ public void maintenancePgCatalog(Connection connexion, String type) {
*/
public Collection<String> getColumns(Connection connexion, Collection<String> liste, String tableIn)
throws ArcException {
liste.addAll(
new GenericBean(executeRequest(connexion, FormatSQL.listeColonneByHeaders(tableIn))).getHeaders());
liste.addAll(new GenericBean(executeRequest(connexion, FormatSQL.listeColonneByHeaders(tableIn))).getHeaders());
return liste;
}

Expand All @@ -757,8 +755,8 @@ public List<String> getColumns(Connection connexion, String tableIn) throws ArcE
* jour "12 as a"
* @throws ArcException
*/
public void fastUpdate(Connection aConnexion, String tableName, String keys,
List<String> colList, String where, String... set) throws ArcException {
public void fastUpdate(Connection aConnexion, String tableName, String keys, List<String> colList, String where,
String... set) throws ArcException {
// récupérer la liste des colonnes
// liste de toutes les colonnes
// liste des colonnes à mettre à jour
Expand Down Expand Up @@ -864,30 +862,14 @@ public void fastUpdate(Connection aConnexion, String tableName, String keys,
* @throws ArcException
* @throws IOException
*/
public void exporting(Connection connexion, String table, OutputStream os, boolean csv, boolean... forceQuote)
throws ArcException {
public void exporting(Connection connexion, String table, OutputStream os, boolean csv) throws ArcException {
ConnectionWrapper conn = initConnection(connexion);

boolean forceQuoteBis;
if (forceQuote != null && forceQuote.length > 0) {
forceQuoteBis = forceQuote[0];
} else {
forceQuoteBis = true;
}

try {
CopyManager copyManager = new CopyManager((BaseConnection) conn.getConnexion());
if (csv) {
if (forceQuoteBis) {
copyManager.copyOut("COPY " + table
+ " TO STDOUT WITH (FORMAT csv, HEADER true , DELIMITER ';' , FORCE_QUOTE *, ENCODING 'UTF8') ",
os);
} else {
copyManager.copyOut(
"COPY " + table
+ " TO STDOUT WITH (FORMAT csv, HEADER true , DELIMITER ';' , ENCODING 'UTF8') ",
os);
}
copyManager.copyOut("COPY " + table
+ " TO STDOUT WITH (FORMAT csv, HEADER false , DELIMITER ';' , ENCODING 'UTF8') ", os);
} else {
copyManager.copyOut("COPY " + table + " TO STDOUT WITH (FORMAT BINARY)", os);
}
Expand Down
17 changes: 9 additions & 8 deletions ...java/fr/insee/arc/ws/services/importServlet/ImportStep1InitializeClientTablesService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import fr.insee.arc.utils.exception.ArcExceptionMessage;
import fr.insee.arc.ws.services.importServlet.actions.SendResponse;
import fr.insee.arc.ws.services.importServlet.bo.ArcClientIdentifier;
import fr.insee.arc.ws.services.importServlet.bo.ExportSource;
import fr.insee.arc.ws.services.importServlet.bo.JsonKeys;
import fr.insee.arc.ws.services.importServlet.dao.ClientDao;

Expand Down Expand Up @@ -45,8 +46,8 @@ public ImportStep1InitializeClientTablesService(JSONObject dsnRequest) {

private List<String> sources;

private void executeIf(String source, Executable exe) throws ArcException {
if (!sources.contains(source)) {
private void executeIf(ExportSource source, Executable exe) throws ArcException {
if (!sources.contains(source.getSource())) {
return;
}
exe.execute();
Expand Down Expand Up @@ -93,13 +94,13 @@ private void startTableCreationInParallel() {
public void run() {
try {
if (tablesMetierNames != null) {
executeIf(ServletArc.MAPPING, () -> clientDao.createImages(tablesMetierNames));
executeIf(ServletArc.METADATA, () -> clientDao.createTableMetier());
executeIf(ServletArc.METADATA, () -> clientDao.createVarMetier());
executeIf(ExportSource.MAPPING, () -> clientDao.createImages(tablesMetierNames));
executeIf(ExportSource.METADATA, () -> clientDao.createTableMetier());
executeIf(ExportSource.METADATA, () -> clientDao.createVarMetier());
}
executeIf(ServletArc.NOMENCLATURE, () -> clientDao.createNmcl());
executeIf(ServletArc.METADATA, () -> clientDao.createTableFamille());
executeIf(ServletArc.METADATA, () -> clientDao.createTablePeriodicite());
executeIf(ExportSource.NOMENCLATURE, () -> clientDao.createNmcl());
executeIf(ExportSource.METADATA, () -> clientDao.createTableFamille());
executeIf(ExportSource.METADATA, () -> clientDao.createTablePeriodicite());
} catch (ArcException e) {
try {
clientDao.createTableWsKO();
Expand Down
4 changes: 2 additions & 2 deletions .../src/main/java/fr/insee/arc/ws/services/importServlet/ImportStep3GetTableDataService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@
import fr.insee.arc.utils.utils.Sleep;
import fr.insee.arc.ws.services.importServlet.actions.SendResponse;
import fr.insee.arc.ws.services.importServlet.bo.ArcClientIdentifier;
import fr.insee.arc.ws.services.importServlet.bo.ExportFormat;
import fr.insee.arc.ws.services.importServlet.dao.ClientDao;
import fr.insee.arc.ws.services.importServlet.dao.ServiceDao;

public class ImportStep3GetTableDataService {

protected static final Logger LOGGER = LogManager.getLogger(ImportStep3GetTableDataService.class);

private static final boolean IS_EXPORT_CSV = false;
// delay to wait before requiring a new table when webservice is still creating tables to retrieve
private static final int WAIT_DELAY_ON_PENDING_TABLES_CREATION_IN_MS = 10000;

Expand All @@ -37,7 +37,7 @@ public void execute(SendResponse resp) throws ArcException {

// binary transfer
ServiceDao.execQueryExportDataToResponse(resp.getWr(),
ViewEnum.normalizeTableName(arcClientIdentifier.getClient()), IS_EXPORT_CSV);
ViewEnum.normalizeTableName(arcClientIdentifier.getClient()), this.arcClientIdentifier.getFormat().equals(ExportFormat.CSV_GZIP.getFormat()));

if (this.clientDao.isWebServiceNotPending()) {
this.clientDao.dropTable(arcClientIdentifier.getClient());
Expand Down
14 changes: 8 additions & 6 deletions arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ServletArc.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,18 +24,16 @@
import fr.insee.arc.utils.webutils.WebUtils;
import fr.insee.arc.ws.services.importServlet.actions.InitiateRequest;
import fr.insee.arc.ws.services.importServlet.actions.SendResponse;
import fr.insee.arc.ws.services.importServlet.bo.ExportFormat;
import fr.insee.arc.ws.services.importServlet.bo.ExportSource;
import fr.insee.arc.ws.services.importServlet.bo.JsonKeys;
import fr.insee.arc.ws.services.importServlet.dao.SecurityDao;

public class ServletArc extends HttpServlet {

private static final long serialVersionUID = 1L;

public static final String NOMENCLATURE = "nomenclature";
public static final String METADATA = "metadata";
public static final String MAPPING = "mapping";

private static final List<String> DEFAULT_SOURCE = Arrays.asList(MAPPING, NOMENCLATURE, METADATA);
private static final List<String> DEFAULT_SOURCE = Arrays.asList(ExportSource.MAPPING.getSource(), ExportSource.NOMENCLATURE.getSource(), ExportSource.METADATA.getSource());

private static final Logger LOGGER = LogManager.getLogger(ServletArc.class);

Expand Down Expand Up @@ -103,7 +101,11 @@ private JSONObject buildRequest(HttpServletRequest request) {

// get parameters from request
JSONObject returned = new JSONObject(request.getParameter("requests"));


if (returned.isNull(JsonKeys.FORMAT.getKey())) {
returned.put(JsonKeys.FORMAT.getKey(), ExportFormat.BINARY.getFormat());
}

// if SOURCE key is not specified, add all the default sources to be retrieved
if (returned.isNull(JsonKeys.SOURCE.getKey())) {
return returned.put(JsonKeys.SOURCE.getKey(), DEFAULT_SOURCE);
Expand Down
75 changes: 3 additions & 72 deletions arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/actions/SendResponse.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,73 +12,15 @@
import fr.insee.arc.utils.exception.ArcException;
import fr.insee.arc.utils.utils.LoggerHelper;



/**Cette classe permet d'envoyer la réponse au client via un flux compressé.
* La réponse aura une des formes suivantes :
*
* Dans le cas d'un appel au service "arcClient" :
* {
* "type":"jsonwsp/response",
* "responses":
* [
* {
* "id":"int",
* "table":
* [
* "nomColonne1;nomColonne2;...;nomColonnen;",
* "typeColonne1;typeColonne2;...;typeColonnen;",
* "a11;a12;...;a1n;",
* "...",
* "an1;an2;...;ann";
* ]
* },
* ],
* "nomenclatures" :
* [
* {
* "id":"int",
* "table":
* [
* "nomColonne1;nomColonne2;...;nomColonnen;",
* "typeColonne1;typeColonne2;...;typeColonnen;",
* "a11;a12;...;a1n;",
* "...",
* "an1;an2;...;ann;"
* ]
* },
* ],
* "varMetier" :
* [
* {
* "id":"int",
* "table":
* [
* "nomColonne1;nomColonne2;...;nomColonnen;",
* "typeColonne1;typeColonne2;...;typeColonnen;",
* "a11;a12;...;a1n;",
* "...",
* "an1;an2;...;ann;"
* ]
* },
* ]
* }
*
* - dans le cas d'une erreur :
* {
* "type":"jsonwsp/response",
* "error":"string"
* }
*
* @author N6YF91
*
/**
* Cette classe permet d'envoyer la réponse au client
*/
public class SendResponse {

private static final Logger LOGGER = LogManager.getLogger(SendResponse.class);

private ServletOutputStream wr;
private HttpServletResponse response;
private HttpServletResponse response;

public SendResponse( HttpServletResponse response ){
this.response = response;
Expand Down Expand Up @@ -125,19 +67,8 @@ public void endSending(){
}
}



public ServletOutputStream getWr() {
return wr;
}



public void setWr(ServletOutputStream wr) {
this.wr = wr;
}




}
Loading

0 comments on commit b90fe54

Please sign in to comment.