diff --git a/arc-core/src/main/java/fr/insee/arc/core/dataobjects/ViewEnum.java b/arc-core/src/main/java/fr/insee/arc/core/dataobjects/ViewEnum.java index 2e80d5bd1..140039519 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/dataobjects/ViewEnum.java +++ b/arc-core/src/main/java/fr/insee/arc/core/dataobjects/ViewEnum.java @@ -150,6 +150,7 @@ public enum ViewEnum { , WS_INFO("ws_info", SchemaEnum.SANDBOX_GENERATED) , WS_PENDING("ws_pending", SchemaEnum.SANDBOX_GENERATED) , WS_KO("ws_ko", SchemaEnum.SANDBOX_GENERATED) + , WS_TRACKING("ws_tracking", SchemaEnum.SANDBOX_GENERATED) ; @@ -205,8 +206,12 @@ public String getFullName(String schema) { return normalizeTableName(schema + SQL.DOT.getSqlCode() + this.tableName); } + public static String getFullNameNotNormalized(String schema, String providedTableName) { + return providedTableName.contains(SQL.DOT.getSqlCode())? providedTableName : schema + SQL.DOT.getSqlCode() + providedTableName; + } + public static String getFullName(String schema, String providedTableName) { - return normalizeTableName(providedTableName.contains(SQL.DOT.getSqlCode())? providedTableName : schema + SQL.DOT.getSqlCode() + providedTableName); + return normalizeTableName(getFullNameNotNormalized(schema, providedTableName)); } public static String normalizeTableName(String providedTableName) diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/global/dao/TableNaming.java b/arc-core/src/main/java/fr/insee/arc/core/service/global/dao/TableNaming.java index 05b725416..a92fb7281 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/global/dao/TableNaming.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/global/dao/TableNaming.java @@ -61,7 +61,7 @@ public static String buildTableNameWithTokens(String schema, String mainSuffix, } s.append(mainSuffix); - return ViewEnum.getFullName(schema, s.toString()); + return ViewEnum.getFullNameNotNormalized(schema, s.toString()); } diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/global/scalability/ServiceScalability.java b/arc-core/src/main/java/fr/insee/arc/core/service/global/scalability/ServiceScalability.java index e819dce32..29daa7f9a 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/global/scalability/ServiceScalability.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/global/scalability/ServiceScalability.java @@ -44,10 +44,8 @@ public static int dispatchOnNods(Connection coordinatorConnexion, ThrowingConsum actionOnCoordinator.accept(coordinatorConnexion); } - + // dispatch when scaled int numberOfExecutorNods = ArcDatabase.numberOfExecutorNods(); - - // meta data copy is only necessary when scaled if (numberOfExecutorNods==0) { return numberOfExecutorNods; diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/metadata/SynchronizeRulesAndMetadataOperation.java b/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/metadata/SynchronizeRulesAndMetadataOperation.java index a321d21fd..1d835169d 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/metadata/SynchronizeRulesAndMetadataOperation.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/metadata/SynchronizeRulesAndMetadataOperation.java @@ -150,7 +150,7 @@ private static void copyMetaDataToExecutors(Connection coordinatorConnexion, Con GenericBean gb = SynchronizeRulesAndMetadataDao.execQuerySelectMetaDataOnlyFrom(coordinatorConnexion, table); - CopyObjectsToDatabase.execCopyFromGenericBeanIfTableNotExists(executorConnection, table, gb); + CopyObjectsToDatabase.execCopyFromGenericBeanWithoutDroppingTargetTable(executorConnection, table, gb); } } diff --git a/arc-utils/src/main/java/fr/insee/arc/utils/dao/CopyObjectsToDatabase.java b/arc-utils/src/main/java/fr/insee/arc/utils/dao/CopyObjectsToDatabase.java index 0f6c3f46f..56678fe76 100644 --- a/arc-utils/src/main/java/fr/insee/arc/utils/dao/CopyObjectsToDatabase.java +++ b/arc-utils/src/main/java/fr/insee/arc/utils/dao/CopyObjectsToDatabase.java @@ -18,35 +18,35 @@ private CopyObjectsToDatabase() { /** * execute copy by chunk. It is mandatory for large GenericBean objects * @param connection - * @param tableName - * @param gb + * @param targetTableName + * @param genericBeanContainingData * @throws ArcException */ - public static void execCopyFromGenericBean(Connection connection, String tableName, GenericBean gb) + public static void execCopyFromGenericBean(Connection connection, String targetTableName, GenericBean genericBeanContainingData) throws ArcException { - execCopyFromGenericBean(connection, tableName, gb, CHUNK_SIZE, true); + execCopyFromGenericBean(connection, targetTableName, genericBeanContainingData, CHUNK_SIZE, true); } - public static void execCopyFromGenericBeanIfTableNotExists(Connection connection, String tableName, GenericBean gb) + public static void execCopyFromGenericBeanWithoutDroppingTargetTable(Connection targetConnection, String targetTableName, GenericBean genericBeanContainingData) throws ArcException { - execCopyFromGenericBean(connection, tableName, gb, CHUNK_SIZE, false); + execCopyFromGenericBean(targetConnection, targetTableName, genericBeanContainingData, CHUNK_SIZE, false); } /** * execute copy from GenericBean to database by chunk of size @param chunkSize * - * @param connection - * @param tableName - * @param gb + * @param targetConnection + * @param targetTableName + * @param genericBeanContainingData * @param chunkSize * @throws ArcException */ - private static void execCopyFromGenericBean(Connection connection, String tableName, GenericBean gb, int chunkSize, boolean replaceTargetTable) + private static void execCopyFromGenericBean(Connection targetConnection, String targetTableName, GenericBean genericBeanContainingData, int chunkSize, boolean replaceTargetTable) throws ArcException { GenericPreparedStatementBuilder query = new GenericPreparedStatementBuilder(); - query.append(query.createWithGenericBean(tableName, gb, replaceTargetTable)); + query.append(query.createWithGenericBean(targetTableName, genericBeanContainingData, replaceTargetTable)); int cursor = 0; boolean stillToDo = true; @@ -55,18 +55,18 @@ private static void execCopyFromGenericBean(Connection connection, String tableN int startChunk = cursor; int endChunk = cursor + chunkSize; cursor = endChunk; - stillToDo=(cursor < gb.getContent().size()); + stillToDo=(cursor < genericBeanContainingData.getContent().size()); - query.insertWithGenericBeanByChunk(tableName, gb, startChunk, endChunk); + query.insertWithGenericBeanByChunk(targetTableName, genericBeanContainingData, startChunk, endChunk); // analyze on the table at the end if (!stillToDo) { query.append(SQL.COMMIT).append(SQL.END_QUERY); - query.append(FormatSQL.analyzeSecured(tableName)); + query.append(FormatSQL.analyzeSecured(targetTableName)); } - UtilitaireDao.get(0).executeImmediate(connection, query); + UtilitaireDao.get(0).executeImmediate(targetConnection, query); query = new GenericPreparedStatementBuilder(); } while (stillToDo); diff --git a/arc-utils/src/main/java/fr/insee/arc/utils/exception/ArcExceptionMessage.java b/arc-utils/src/main/java/fr/insee/arc/utils/exception/ArcExceptionMessage.java index bbdff899b..1058776fe 100644 --- a/arc-utils/src/main/java/fr/insee/arc/utils/exception/ArcExceptionMessage.java +++ b/arc-utils/src/main/java/fr/insee/arc/utils/exception/ArcExceptionMessage.java @@ -87,6 +87,7 @@ public enum ArcExceptionMessage { WS_RETRIEVE_DATA_FAMILY_FORBIDDEN("Vous ne pouvez pas accéder à cette famille de norme"), WS_RETRIEVE_DATA_FAMILY_CREATION_FAILED("Les tables de la famille de norme n'ont pas pu être créées"), + WS_RETRIEVE_DATA_SCALABLE_TABLE_MUST_BE_EXPORT_IN_CSV("Scalable tables can only be retrieved in csv_gzip mode"), IHM_NMCL_COLUMN_IN_FILE_BUT_NOT_IN_SCHEMA("La colonne %s n'est pas déclarée dans le schéma"), IHM_NMCL_COLUMN_IN_SCHEMA_BUT_NOT_IN_FILE("La colonne est déclarée dans le schéma mais absente du fichier"), diff --git a/arc-ws/pom.xml b/arc-ws/pom.xml index 8a3208bc1..662ef3772 100644 --- a/arc-ws/pom.xml +++ b/arc-ws/pom.xml @@ -19,6 +19,20 @@ + + fr.insee.arc + arc-utils + ${project.version} + test-jar + test + + + fr.insee.arc + arc-core + ${project.version} + test-jar + test + fr.insee.arc arc-utils diff --git a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ImportStep1InitializeClientTablesService.java b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ImportStep1InitializeClientTablesService.java index d851e5262..614454c2f 100644 --- a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ImportStep1InitializeClientTablesService.java +++ b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ImportStep1InitializeClientTablesService.java @@ -8,6 +8,7 @@ import org.json.JSONArray; import org.json.JSONObject; +import fr.insee.arc.core.dataobjects.ArcDatabase; import fr.insee.arc.core.dataobjects.SchemaEnum; import fr.insee.arc.core.model.Delimiters; import fr.insee.arc.utils.exception.ArcException; @@ -36,7 +37,7 @@ public ImportStep1InitializeClientTablesService(JSONObject dsnRequest) { this.dsnRequest = dsnRequest; - this.arcClientIdentifier = new ArcClientIdentifier(dsnRequest); + this.arcClientIdentifier = new ArcClientIdentifier(dsnRequest, true); this.sources = makeSource(dsnRequest); @@ -63,25 +64,68 @@ private static List makeSource(JSONObject dsnRequest) { } public void execute(SendResponse resp) throws ArcException { - this.clientDao.dropPendingClientTables(); + + // drop tables from the client that had been requested from a former call + dropPendingClientTables(); - this.clientDao.createTableWsStatus(); + // create the table that will track the data table which has been built and retrieved + createTrackTable(); + + // create the wsinfo and the wspending table + // wspending table will be delete when all + createWsTables(); - if (!arcClientIdentifier.getEnvironnement().equalsIgnoreCase(SchemaEnum.ARC_METADATA.getSchemaName())) { - clientDao.verificationClientFamille(); - tablesMetierNames = clientDao.getIdSrcTableMetier(dsnRequest); - } + // create tables to retrieve family data table + createMetaFamilyTables(); + // create data table in an asynchronous parallel thread startTableCreationInParallel(); // on renvoie l'id du client avec son timestamp resp.send(arcClientIdentifier.getEnvironnement() + Delimiters.SQL_SCHEMA_DELIMITER - + arcClientIdentifier.getClient() + Delimiters.SQL_TOKEN_DELIMITER + + arcClientIdentifier.getClientIdentifier() + Delimiters.SQL_TOKEN_DELIMITER + arcClientIdentifier.getTimestamp()); resp.endSending(); } + /** + * 1. check if the client has the right to retrieve the family. If so : + * 2. build the table of id_source to be retrieved in the family data table + * 3. return the list of family data table to retrieve + * @throws ArcException + */ + private void createMetaFamilyTables() throws ArcException { + if (!arcClientIdentifier.getEnvironnement().equalsIgnoreCase(SchemaEnum.ARC_METADATA.getSchemaName())) { + + if (!clientDao.verificationClientFamille()) { + throw new ArcException(ArcExceptionMessage.WS_RETRIEVE_DATA_FAMILY_FORBIDDEN); + } + + clientDao.createTableOfIdSource(dsnRequest); + tablesMetierNames = clientDao.selectBusinessDataTables(); + } + } + + /** + * create the table that tracks the client table which had been built + * when the data of a table will be retrieved by the client, the table entry will be deleted from the track table + * @throws ArcException + */ + private void createTrackTable() throws ArcException { + clientDao.createTableTrackRetrievedTables(); + } + + /** + * create the wsinfo and wspending tables + * wspending will be deleted when all client tables will have been retrieved + * wsinfo table will be looped transfered to the client until wspending table is dropped + * @throws ArcException + */ + private void createWsTables() throws ArcException { + this.clientDao.createTableWsInfo(); + } + /** * Will send handshake to client every @HANDSHAKE_TIMER_IN_MS milliseconds Ugly * but we failed at fixing that in front of a F5 controller @@ -94,11 +138,12 @@ private void startTableCreationInParallel() { public void run() { try { if (tablesMetierNames != null) { - executeIf(ExportSource.MAPPING, () -> clientDao.createImages(tablesMetierNames)); + + executeIf(ExportSource.MAPPING, () -> createImages(tablesMetierNames)); executeIf(ExportSource.METADATA, () -> clientDao.createTableMetier()); - executeIf(ExportSource.METADATA, () -> clientDao.createVarMetier()); + executeIf(ExportSource.METADATA, () -> clientDao.createTableVarMetier()); } - executeIf(ExportSource.NOMENCLATURE, () -> clientDao.createNmcl()); + executeIf(ExportSource.NOMENCLATURE, () -> clientDao.createTableNmcl()); executeIf(ExportSource.METADATA, () -> clientDao.createTableFamille()); executeIf(ExportSource.METADATA, () -> clientDao.createTablePeriodicite()); } catch (ArcException e) { @@ -125,4 +170,46 @@ public void run() { maintenance.start(); } + + /** + * drop tables on coordinator and executors if the exists + * @throws ArcException + */ + private void dropPendingClientTables() throws ArcException { + + this.clientDao.dropPendingClientTables(0); + + int numberOfExecutorNods = ArcDatabase.numberOfExecutorNods(); + for (int executorConnectionId = ArcDatabase.EXECUTOR.getIndex(); executorConnectionId < ArcDatabase.EXECUTOR + .getIndex() + numberOfExecutorNods; executorConnectionId++) { + this.clientDao.dropPendingClientTables(executorConnectionId); + } + } + + + + /** + * create image tables on executor nods if connection is scaled, on coordinator + * nod if not + * + * @param tablesMetierNames + * @throws ArcException + */ + private void createImages(List tablesMetierNames) throws ArcException { + int numberOfExecutorNods = ArcDatabase.numberOfExecutorNods(); + if (numberOfExecutorNods == 0) { + clientDao.createImages(tablesMetierNames, ArcDatabase.COORDINATOR.getIndex()); + } else { + for (int executorConnectionId = ArcDatabase.EXECUTOR.getIndex(); executorConnectionId < ArcDatabase.EXECUTOR + .getIndex() + numberOfExecutorNods; executorConnectionId++) { + + // copy the table containing id_source to be retrieved on executor nods + clientDao.copyTableOfIdSourceToExecutorNod(executorConnectionId); + + // create the business table containing data of id_source found in table tableOfIdSource + clientDao.createImages(tablesMetierNames, executorConnectionId); + } + } + } + } diff --git a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ImportStep2GetTableNameService.java b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ImportStep2GetTableNameService.java index 9610a9727..8409fd25f 100644 --- a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ImportStep2GetTableNameService.java +++ b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ImportStep2GetTableNameService.java @@ -7,9 +7,12 @@ import org.json.JSONObject; import fr.insee.arc.utils.exception.ArcException; +import fr.insee.arc.utils.exception.ArcExceptionMessage; import fr.insee.arc.ws.services.importServlet.actions.SendResponse; import fr.insee.arc.ws.services.importServlet.bo.ArcClientIdentifier; +import fr.insee.arc.ws.services.importServlet.bo.ExportTrackingType; import fr.insee.arc.ws.services.importServlet.bo.JsonKeys; +import fr.insee.arc.ws.services.importServlet.bo.TableToRetrieve; import fr.insee.arc.ws.services.importServlet.dao.ClientDao; import fr.insee.arc.ws.services.importServlet.dao.NameDao; @@ -29,7 +32,7 @@ public ImportStep2GetTableNameService(JSONObject dsnRequest) { this.dsnRequest = dsnRequest; - this.arcClientIdentifier = new ArcClientIdentifier(dsnRequest); + this.arcClientIdentifier = new ArcClientIdentifier(dsnRequest, false); reprise = this.dsnRequest.getBoolean(JsonKeys.REPRISE.getKey()); @@ -39,26 +42,20 @@ public ImportStep2GetTableNameService(JSONObject dsnRequest) { public void execute(SendResponse resp) throws ArcException { - StringBuilder type = new StringBuilder(); - - String tableName = this.clientDao.getAClientTable(); - - if (tableName == null) { - tableName = this.clientDao.getIdTable(); - - if (!reprise) { - this.clientDao.updatePilotage(tableName); - } + // check if a KO + if (this.clientDao.getAClientTableByType(ExportTrackingType.KO).getTableName() != null) { + throw new ArcException(ArcExceptionMessage.WS_RETRIEVE_DATA_FAMILY_CREATION_FAILED); + } + + // try to get a data table + TableToRetrieve table = this.clientDao.getAClientTableByType(ExportTrackingType.DATA); - this.clientDao.dropTable(tableName); + if (table.getTableName() != null) { - resp.send(" "); - resp.endSending(); - return; + StringBuilder type = new StringBuilder(); - } else { // récupération du type - List> metadataOnlyTable = NameDao.execQuerySelectMetadata(tableName); + List> metadataOnlyTable = NameDao.execQuerySelectMetadata(table); for (int j = 0; j < metadataOnlyTable.get(0).size(); j++) { if (j > 0) { @@ -69,11 +66,31 @@ public void execute(SendResponse resp) throws ArcException { type.append(" " + metadataOnlyTable.get(i).get(j)); } } + + // renvoie un nom de table du client si il en reste une + resp.send(table.getTableName() + " " + type); + resp.endSending(); + + return; } - // renvoie un nom de table du client si il en reste une - resp.send(tableName + " " + type); + // if no data table found, get source table to register + table = this.clientDao.getAClientTableByType(ExportTrackingType.ID_SOURCE); + + if (table.getTableName() != null) { + if (!reprise) { + this.clientDao.updatePilotage(table.getTableName()); + } + + this.clientDao.dropTable(table.getTableName()); + } + + table = this.clientDao.getAClientTableByType(ExportTrackingType.TRACK); + this.clientDao.dropTable(table.getTableName()); + + resp.send(" "); resp.endSending(); + } } diff --git a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ImportStep3GetTableDataService.java b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ImportStep3GetTableDataService.java index c1f8e325d..f848750b0 100644 --- a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ImportStep3GetTableDataService.java +++ b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/ImportStep3GetTableDataService.java @@ -10,6 +10,7 @@ import fr.insee.arc.ws.services.importServlet.actions.SendResponse; import fr.insee.arc.ws.services.importServlet.bo.ArcClientIdentifier; import fr.insee.arc.ws.services.importServlet.bo.ExportFormat; +import fr.insee.arc.ws.services.importServlet.bo.TableToRetrieve; import fr.insee.arc.ws.services.importServlet.dao.ClientDao; import fr.insee.arc.ws.services.importServlet.dao.ServiceDao; @@ -27,7 +28,7 @@ public class ImportStep3GetTableDataService { public ImportStep3GetTableDataService(JSONObject dsnRequest) { super(); - this.arcClientIdentifier = new ArcClientIdentifier(dsnRequest); + this.arcClientIdentifier = new ArcClientIdentifier(dsnRequest, false); clientDao = new ClientDao(arcClientIdentifier); @@ -35,12 +36,14 @@ public ImportStep3GetTableDataService(JSONObject dsnRequest) { public void execute(SendResponse resp) throws ArcException { + TableToRetrieve table = clientDao.getAClientTableByName(arcClientIdentifier.getClientInputParameter()); + // binary transfer - ServiceDao.execQueryExportDataToResponse(resp.getWr(), - ViewEnum.normalizeTableName(arcClientIdentifier.getClient()), ExportFormat.isCsv(this.arcClientIdentifier.getFormat())); + ServiceDao.execQueryExportDataToResponse(resp.getWr(), table, ExportFormat.isCsv(this.arcClientIdentifier.getFormat())); if (this.clientDao.isWebServiceNotPending()) { - this.clientDao.dropTable(arcClientIdentifier.getClient()); + this.clientDao.dropTable(table); + this.clientDao.deleteFromTrackTable(table.getTableName()); } else { Sleep.sleep(WAIT_DELAY_ON_PENDING_TABLES_CREATION_IN_MS); } diff --git a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/ArcClientIdentifier.java b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/ArcClientIdentifier.java index 2e2f31ae6..1542ca8f9 100644 --- a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/ArcClientIdentifier.java +++ b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/ArcClientIdentifier.java @@ -4,41 +4,56 @@ import org.json.JSONObject; +import fr.insee.arc.core.model.Delimiters; import fr.insee.arc.core.service.global.util.Patch; public class ArcClientIdentifier { - - public ArcClientIdentifier(JSONObject dsnRequest) { + + public ArcClientIdentifier(JSONObject dsnRequest, boolean generateTimeStamp) { this.dsnRequest = dsnRequest; - this.client = getKeyIfExists(JsonKeys.CLIENT); - this.timestamp = System.currentTimeMillis(); - this.environnement = getKeyIfExists(JsonKeys.ENVIRONNEMENT, Patch::normalizeSchemaName); + this.clientInputParameter = dsnRequest.getString(JsonKeys.CLIENT.getKey()); + + if (generateTimeStamp) + { + this.clientIdentifier = this.clientInputParameter; + this.timestamp = System.currentTimeMillis(); + this.environnement = getKeyIfExists(JsonKeys.ENVIRONNEMENT, Patch::normalizeSchemaName); + } + else + { + // as example : arc_bas1.ARTEMIS_1701299079078 + String[] tokens = this.clientInputParameter.split("\\"+Delimiters.SQL_SCHEMA_DELIMITER); + this.environnement = tokens[0]; + this.clientIdentifier = tokens[1]; + tokens = this.clientIdentifier.split("\\"+Delimiters.SQL_TOKEN_DELIMITER); + this.clientIdentifier = tokens[0]; + this.timestamp = Long.parseLong(tokens[1]); + } this.famille = getKeyIfExists(JsonKeys.FAMILLE); this.format = getKeyIfExists(JsonKeys.FORMAT); } private JSONObject dsnRequest; + private String clientInputParameter; + private long timestamp; private String environnement; - private String client; + private String clientIdentifier; private String famille; - - private String format; + private String format; - private String getKeyIfExists(JsonKeys key, UnaryOperator f ) - { - return dsnRequest.keySet().contains(key.getKey())?f.apply(dsnRequest.getString(key.getKey())):null; + private String getKeyIfExists(JsonKeys key, UnaryOperator f) { + return dsnRequest.keySet().contains(key.getKey()) ? f.apply(dsnRequest.getString(key.getKey())) : null; } - - private String getKeyIfExists(JsonKeys key) - { - return getKeyIfExists(key, t -> t ); + + private String getKeyIfExists(JsonKeys key) { + return getKeyIfExists(key, t -> t); } public long getTimestamp() { @@ -49,9 +64,8 @@ public String getEnvironnement() { return environnement; } - - public String getClient() { - return client; + public String getClientIdentifier() { + return clientIdentifier; } public String getFamille() { @@ -61,5 +75,9 @@ public String getFamille() { public String getFormat() { return format; } - + + public String getClientInputParameter() { + return clientInputParameter; + } + } diff --git a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/ExportTrackingType.java b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/ExportTrackingType.java new file mode 100644 index 000000000..dd6dc97ee --- /dev/null +++ b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/ExportTrackingType.java @@ -0,0 +1,7 @@ +package fr.insee.arc.ws.services.importServlet.bo; + +public enum ExportTrackingType { + + ID_SOURCE, KO, DATA, TRACK; + +} diff --git a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/TableToRetrieve.java b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/TableToRetrieve.java new file mode 100644 index 000000000..26f61f00c --- /dev/null +++ b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/bo/TableToRetrieve.java @@ -0,0 +1,29 @@ +package fr.insee.arc.ws.services.importServlet.bo; + +import fr.insee.arc.core.dataobjects.ArcDatabase; + +public class TableToRetrieve { + + private ArcDatabase nod; + + private String tableName; + + public TableToRetrieve() { + super(); + } + + public TableToRetrieve(String nod, String tableName) { + super(); + this.nod = ArcDatabase.valueOf(nod); + this.tableName = tableName; + } + + public ArcDatabase getNod() { + return nod; + } + + public String getTableName() { + return tableName; + } + +} diff --git a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/dao/ClientDao.java b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/dao/ClientDao.java index 54156db49..e538406aa 100644 --- a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/dao/ClientDao.java +++ b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/dao/ClientDao.java @@ -1,13 +1,16 @@ package fr.insee.arc.ws.services.importServlet.dao; import java.sql.Connection; +import java.sql.SQLException; import java.util.List; +import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.json.JSONObject; +import fr.insee.arc.core.dataobjects.ArcDatabase; import fr.insee.arc.core.dataobjects.ArcPreparedStatementBuilder; import fr.insee.arc.core.dataobjects.ColumnEnum; import fr.insee.arc.core.dataobjects.ViewEnum; @@ -15,6 +18,8 @@ import fr.insee.arc.core.model.TraitementEtat; import fr.insee.arc.core.model.TraitementPhase; import fr.insee.arc.core.service.global.dao.TableNaming; +import fr.insee.arc.utils.dao.CopyObjectsToDatabase; +import fr.insee.arc.utils.dao.SQL; import fr.insee.arc.utils.dao.UtilitaireDao; import fr.insee.arc.utils.exception.ArcException; import fr.insee.arc.utils.exception.ArcExceptionMessage; @@ -23,7 +28,9 @@ import fr.insee.arc.utils.utils.LoggerHelper; import fr.insee.arc.utils.utils.ManipString; import fr.insee.arc.ws.services.importServlet.bo.ArcClientIdentifier; +import fr.insee.arc.ws.services.importServlet.bo.ExportTrackingType; import fr.insee.arc.ws.services.importServlet.bo.JsonKeys; +import fr.insee.arc.ws.services.importServlet.bo.TableToRetrieve; public class ClientDao { @@ -34,21 +41,32 @@ public class ClientDao { private String client; private String famille; + // the tablename of the table that contains document data to retrieve identified + // by id_source private String tableOfIdSource; + + // the tablename of the table that shows webservice is still creating table to + // be consumed by the client + // it is dropped when client had built all the data table private String tableWsPending; - Connection connection; + // the tablename of the table that tracks tables left to retrieved + private String tableWsTracking; + + private Connection connection; public ClientDao(ArcClientIdentifier arcClientIdentifier) { this.timestamp = arcClientIdentifier.getTimestamp(); this.environnement = arcClientIdentifier.getEnvironnement(); - this.client = arcClientIdentifier.getClient(); + this.client = arcClientIdentifier.getClientIdentifier(); this.famille = arcClientIdentifier.getFamille(); - this.tableOfIdSource = TableNaming.buildTableNameWithTokens(environnement, - ViewEnum.ID_SOURCE, client, timestamp); - this.tableWsPending = TableNaming.buildTableNameWithTokens(environnement, - ViewEnum.WS_PENDING, client, timestamp); + this.tableOfIdSource = TableNaming.buildTableNameWithTokens(environnement, ViewEnum.ID_SOURCE, client, + timestamp); + this.tableWsPending = TableNaming.buildTableNameWithTokens(environnement, ViewEnum.WS_PENDING, client, + timestamp); + this.tableWsTracking = TableNaming.buildTableNameWithTokens(environnement, ViewEnum.WS_TRACKING, client, + timestamp); } @@ -56,7 +74,7 @@ public ClientDao(ArcClientIdentifier arcClientIdentifier) { * Vérifie que le client peut consulter les tables métiers de la famille de * normes */ - public void verificationClientFamille() throws ArcException { + public boolean verificationClientFamille() throws ArcException { LoggerHelper.debugAsComment(LOGGER, timestamp, "ClientDaoImpl#verificationClientFamille()"); ArcPreparedStatementBuilder request = new ArcPreparedStatementBuilder(); @@ -66,52 +84,108 @@ public void verificationClientFamille() throws ArcException { String bool = UtilitaireDao.get(0).executeRequestWithoutMetadata(connection, request).get(0).get(0); - if (!bool.equals("t")) { - throw new ArcException(ArcExceptionMessage.WS_RETRIEVE_DATA_FAMILY_FORBIDDEN); - } + return bool.equals("t"); } /** - * Créer une image des ids sources répondants aux critères et récupère la liste - * des noms des tables métiers + * return the list of business data table related to the famille provided * - * @param JSONObject contient les paramètres de la requête - * @return La liste des noms des tables métiers. + * @return * @throws ArcException */ - public List getIdSrcTableMetier(JSONObject requeteJSON) throws ArcException { + public List selectBusinessDataTables() throws ArcException { - LoggerHelper.debugAsComment(LOGGER, timestamp, "ClientDaoImpl#getIdSrcTableMetier()"); + ArcPreparedStatementBuilder request = new ArcPreparedStatementBuilder(); + request.append("SELECT " + ColumnEnum.NOM_TABLE_METIER + " "); + request.append("FROM " + ViewEnum.MOD_TABLE_METIER.getFullName(environnement) + " T1 "); + request.append("WHERE T1.id_famille='" + this.famille + "' "); + request.append(";"); - // Initialisation des variables + return new GenericBean(UtilitaireDao.get(0).executeRequest(connection, request)) + .getColumnValues(ColumnEnum.NOM_TABLE_METIER.getColumnName()); + } - // Préparation du block de requêtes à executer + private void registerTableToBeRetrieved(ExportTrackingType wsTrackingType, ArcDatabase targetNod, String nomTable) + throws ArcException { + ArcPreparedStatementBuilder query = new ArcPreparedStatementBuilder(); + query.build(SQL.INSERT_INTO, this.tableWsTracking, "(tracking_type, nod, table_to_retrieve)"); + query.build(SQL.SELECT, query.quoteText(wsTrackingType.toString()), ",", query.quoteText(targetNod.toString()), + ",", query.quoteText(nomTable)); + UtilitaireDao.get(0).executeRequest(connection, query); + } - // Création de la requête de création de la table temporaire contenant la liste - // des id_sources - execQueryCreateTableOfIdSource(requeteJSON); + /** + * Créer une image des tables métiers. + * + * @param tablesMetierNames La liste des noms des tables métiers. + * + * @return liste des noms de tables images crées + * @throws ArcException + */ + private void addImage(String tableMetier, int executorConnectionId) throws ArcException { + StringBuilder request = new StringBuilder(); + + String nomTableImage = TableNaming.buildTableNameWithTokens(environnement, tableMetier, client, timestamp); + + request.append("DROP TABLE IF EXISTS " + nomTableImage + "; "); + + request.append("CREATE TABLE " + nomTableImage + FormatSQL.WITH_NO_VACUUM + " AS "); + request.append("SELECT * "); + request.append("FROM " + ViewEnum.getFullName(environnement, tableMetier) + " T1 WHERE true "); + request.append("AND exists (SELECT 1 FROM " + tableOfIdSource + " T2 where T2." + + ColumnEnum.ID_SOURCE.getColumnName() + "=T1." + ColumnEnum.ID_SOURCE.getColumnName() + "); "); + + UtilitaireDao.get(executorConnectionId).executeBlock(connection, request); + + registerTableToBeRetrieved(ExportTrackingType.DATA, ArcDatabase.EXECUTOR, nomTableImage); - return execQuerySelectBusinessDataTables(); - } /** - * return the list of business data table related to the famille provided - * @return + * Met à jours les colonnes client et date_client de la table + * environnement_pilotage_fichier. + * + * @param tableSource * @throws ArcException */ - private List execQuerySelectBusinessDataTables() throws ArcException { + public void updatePilotage(String tableSource) throws ArcException { + LoggerHelper.debugAsComment(LOGGER, timestamp, ": ClientDaoImpl.updatePilotage()"); - ArcPreparedStatementBuilder request = new ArcPreparedStatementBuilder(); - request.append("SELECT "+ColumnEnum.NOM_TABLE_METIER +" "); - request.append("FROM " + ViewEnum.MOD_TABLE_METIER.getFullName(environnement) + " T1 "); - request.append("WHERE T1.id_famille='" + this.famille + "' "); - request.append("AND exists (select 1 from pg_tables T2 where "); - request.append("T2.schemaname='" + ManipString.substringBeforeFirst(environnement, ".") + "' "); - request.append("AND T1.nom_table_metier=T2.tablename);"); + String clientOfTableSource = extractClientFromToken(); - return new GenericBean(UtilitaireDao.get(0).executeRequest(connection, request)).getColumnValues(ColumnEnum.NOM_TABLE_METIER.getColumnName()); + StringBuilder query = new StringBuilder(); + query.append("UPDATE " + ViewEnum.PILOTAGE_FICHIER.getFullName(environnement) + " T1 "); + query.append("SET client = array_append(client, '" + clientOfTableSource + "') "); + query.append(", date_client = array_append( date_client, localtimestamp ) "); + query.append("WHERE true "); + query.append("AND EXISTS (SELECT 1 FROM " + tableSource + " T2 where T1." + ColumnEnum.ID_SOURCE.getColumnName() + + "=T2." + ColumnEnum.ID_SOURCE.getColumnName() + ") "); + query.append("AND T1.phase_traitement='" + TraitementPhase.MAPPING + "';"); + + UtilitaireDao.get(0).executeBlock(connection, query.toString()); + } + + /** + * extract the client token name from the client + * + * @param client2 + * @return + */ + private String extractClientFromToken() { + return ManipString.substringBeforeFirst( + ManipString.substringAfterFirst(this.client, Delimiters.SQL_SCHEMA_DELIMITER), + Delimiters.SQL_TOKEN_DELIMITER); + } + + public void createTableTrackRetrievedTables() throws ArcException { + ArcPreparedStatementBuilder query = new ArcPreparedStatementBuilder(); + query.build(SQL.DROP, SQL.TABLE, SQL.IF_EXISTS, this.tableWsTracking, SQL.END_QUERY); + query.build(SQL.CREATE, SQL.TABLE, this.tableWsTracking, + " (tracking_type text, nod text, table_to_retrieve text) ", SQL.END_QUERY); + UtilitaireDao.get(0).executeRequest(connection, query); + + registerTableToBeRetrieved(ExportTrackingType.TRACK, ArcDatabase.COORDINATOR, this.tableWsTracking); } /** @@ -124,9 +198,9 @@ private List execQuerySelectBusinessDataTables() throws ArcException { * * @param query * @param requeteJSON - * @throws ArcException + * @throws ArcException */ - private void execQueryCreateTableOfIdSource(JSONObject requeteJSON) throws ArcException { + public void createTableOfIdSource(JSONObject requeteJSON) throws ArcException { String periodicite = requeteJSON.getString(JsonKeys.PERIODICITE.getKey()); String validiteInf = requeteJSON.keySet().contains(JsonKeys.VALINF.getKey()) @@ -176,9 +250,11 @@ private void execQueryCreateTableOfIdSource(JSONObject requeteJSON) throws ArcEx query.append(nbFichiers); } query.append(") as foo; "); - + UtilitaireDao.get(0).executeBlock(connection, query); + registerTableToBeRetrieved(ExportTrackingType.ID_SOURCE, ArcDatabase.EXECUTOR, tableOfIdSource); + } /** @@ -189,82 +265,21 @@ private void execQueryCreateTableOfIdSource(JSONObject requeteJSON) throws ArcEx * @return liste des noms de tables images crées * @throws ArcException */ - public void createImages(List tablesMetierNames) throws ArcException { + public void createImages(List tablesMetierNames, int executorConnectionId) throws ArcException { LoggerHelper.debugAsComment(LOGGER, timestamp, "ClientDaoImpl.createImage()"); for (String tableMetier : tablesMetierNames) { - addImage(tableMetier); + addImage(tableMetier, executorConnectionId); } } - /** - * Créer une image des tables métiers. - * - * @param tablesMetierNames La liste des noms des tables métiers. - * - * @return liste des noms de tables images crées - * @throws ArcException - */ - public void addImage(String tableMetier) throws ArcException { - StringBuilder request = new StringBuilder(); - - String nomTableImage = TableNaming.buildTableNameWithTokens(environnement, tableMetier, client, timestamp); - - request.append("DROP TABLE IF EXISTS " + nomTableImage + "; "); - - request.append("CREATE TABLE " + nomTableImage + FormatSQL.WITH_NO_VACUUM + " AS "); - request.append("SELECT * "); - request.append("FROM " + ViewEnum.getFullName(environnement, tableMetier) + " T1 WHERE true "); - request.append("AND exists (SELECT 1 FROM " + tableOfIdSource + " T2 where T2." - + ColumnEnum.ID_SOURCE.getColumnName() + "=T1." + ColumnEnum.ID_SOURCE.getColumnName() + "); "); - - UtilitaireDao.get(0).executeBlock(connection, request); - - } - - /** - * Met à jours les colonnes client et date_client de la table - * environnement_pilotage_fichier. - * - * @param tableSource - * @throws ArcException - */ - public void updatePilotage(String tableSource) throws ArcException { - LoggerHelper.debugAsComment(LOGGER, timestamp, ": ClientDaoImpl.updatePilotage()"); - - String clientOfTableSource = extractClientFromToken(); - - StringBuilder query = new StringBuilder(); - query.append("UPDATE " + ViewEnum.PILOTAGE_FICHIER.getFullName(environnement) + " T1 "); - query.append("SET client = array_append(client, '" + clientOfTableSource + "') "); - query.append(", date_client = array_append( date_client, localtimestamp ) "); - query.append("WHERE true "); - query.append("AND EXISTS (SELECT 1 FROM " + tableSource + " T2 where T1." - + ColumnEnum.ID_SOURCE.getColumnName() + "=T2." + ColumnEnum.ID_SOURCE.getColumnName() + ") "); - query.append("AND T1.phase_traitement='" + TraitementPhase.MAPPING + "';"); - - UtilitaireDao.get(0).executeBlock(connection, query.toString()); - } - - /** - * extract the client token name from the client - * @param client2 - * @return - */ - private String extractClientFromToken() { - return - ManipString.substringBeforeFirst( - ManipString.substringAfterFirst(this.client, Delimiters.SQL_SCHEMA_DELIMITER), - Delimiters.SQL_TOKEN_DELIMITER); - } - /* * (non-Javadoc) * * @see * fr.insee.arc_essnet.ws.dao.ClientDarcl(fr.insee.arc_essnet.ws.actions.Senarc */ - public void createNmcl() throws ArcException { + public void createTableNmcl() throws ArcException { LoggerHelper.debugAsComment(LOGGER, "ClientDaoImpl.createNmcl()"); ArcPreparedStatementBuilder requete = new ArcPreparedStatementBuilder(); @@ -275,9 +290,10 @@ public void createNmcl() throws ArcException { List> nmclNames = UtilitaireDao.get(0).executeRequestWithoutMetadata(connection, requete); for (List nmcl : nmclNames) { - String nomTableImage = ViewEnum.getFullName(environnement, client + "_" + timestamp + "_" + nmcl.get(0)); + String nomTableImage = ViewEnum.getFullNameNotNormalized(environnement, client + "_" + timestamp + "_" + nmcl.get(0)); UtilitaireDao.get(0).executeImmediate(connection, "CREATE TABLE " + nomTableImage + FormatSQL.WITH_NO_VACUUM + " AS SELECT * FROM " + ViewEnum.getFullName(environnement, nmcl.get(0)) + ";"); + registerTableToBeRetrieved(ExportTrackingType.DATA, ArcDatabase.COORDINATOR, nomTableImage); } } @@ -288,11 +304,11 @@ public void createNmcl() throws ArcException { * @see fr.insee.arc_essnet.ws.dao.ClientDarcMetier(java.lang.String, * fr.insee.arc_essnet.ws.actions.Senarc */ - public void createVarMetier() throws ArcException { + public void createTableVarMetier() throws ArcException { LoggerHelper.debugAsComment(LOGGER, "ClientDaoImpl.createVarMetier()"); - String nomTableImage = TableNaming.buildTableNameWithTokens(environnement, - ViewEnum.MOD_VARIABLE_METIER, client, timestamp); + String nomTableImage = TableNaming.buildTableNameWithTokens(environnement, ViewEnum.MOD_VARIABLE_METIER, client, + timestamp); ArcPreparedStatementBuilder requete = new ArcPreparedStatementBuilder(); requete.append("CREATE TABLE " + nomTableImage + FormatSQL.WITH_NO_VACUUM + " AS"); @@ -301,6 +317,8 @@ public void createVarMetier() throws ArcException { requete.append(";"); UtilitaireDao.get(0).executeRequest(connection, requete); + registerTableToBeRetrieved(ExportTrackingType.DATA, ArcDatabase.COORDINATOR, nomTableImage); + } /* @@ -312,9 +330,9 @@ public void createVarMetier() throws ArcException { public void createTableFamille() throws ArcException { LoggerHelper.debugAsComment(LOGGER, "ClientDaoImpl.createTableFamille()"); - String nomTableImage = TableNaming.buildTableNameWithTokens(environnement, - ViewEnum.EXT_MOD_FAMILLE, client, timestamp); - + String nomTableImage = TableNaming.buildTableNameWithTokens(environnement, ViewEnum.EXT_MOD_FAMILLE, client, + timestamp); + ArcPreparedStatementBuilder requete = new ArcPreparedStatementBuilder(); requete.append("CREATE TABLE " + nomTableImage + FormatSQL.WITH_NO_VACUUM + " AS SELECT DISTINCT f.id_famille FROM arc.ihm_famille f INNER JOIN " @@ -322,6 +340,8 @@ public void createTableFamille() throws ArcException { + requete.quoteText(client) + ");"); UtilitaireDao.get(0).executeRequest(connection, requete); + registerTableToBeRetrieved(ExportTrackingType.ID_SOURCE, ArcDatabase.COORDINATOR, nomTableImage); + } /* @@ -339,6 +359,8 @@ public void createTablePeriodicite() throws ArcException { UtilitaireDao.get(0).executeImmediate(connection, "CREATE TABLE " + nomTableImage + FormatSQL.WITH_NO_VACUUM + " AS SELECT DISTINCT id, val FROM " + ViewEnum.EXT_MOD_PERIODICITE.getFullName() + ";"); + registerTableToBeRetrieved(ExportTrackingType.DATA, ArcDatabase.COORDINATOR, nomTableImage); + } /* @@ -350,7 +372,8 @@ public void createTablePeriodicite() throws ArcException { public void createTableMetier() throws ArcException { LoggerHelper.debugAsComment(LOGGER, "ClientDaoImpl.sendTableMetier()"); - String nomTableImage = TableNaming.buildTableNameWithTokens(environnement, ViewEnum.MOD_TABLE_METIER, client, timestamp); + String nomTableImage = TableNaming.buildTableNameWithTokens(environnement, ViewEnum.MOD_TABLE_METIER, client, + timestamp); ArcPreparedStatementBuilder requete = new ArcPreparedStatementBuilder( "\n CREATE TABLE " + nomTableImage + FormatSQL.WITH_NO_VACUUM + " AS"); @@ -359,62 +382,77 @@ public void createTableMetier() throws ArcException { requete.append(";"); UtilitaireDao.get(0).executeRequest(connection, requete); + registerTableToBeRetrieved(ExportTrackingType.DATA, ArcDatabase.COORDINATOR, nomTableImage); } /** + * Get the table object of the table to retrieve by its type * - * @param client - * @param isSourceListTable : is it the table containing the list of id_source - * of the files to be marked ? + * @param tableName * @return * @throws ArcException */ - public String getAClientTable(boolean isSourceListTable) throws ArcException { - - String schema = ManipString.substringBeforeFirst(client, "."); - String tableToFind = ViewEnum.normalizeTableName(ManipString.substringAfterFirst(client, ".").replace("_", "\\_") + "%"); - String tableWsInfo = ViewEnum.normalizeTableName(this.client + Delimiters.SQL_TOKEN_DELIMITER + ViewEnum.WS_INFO.getTableName()); - String tableWsKO = ViewEnum.normalizeTableName(this.client + Delimiters.SQL_TOKEN_DELIMITER + ViewEnum.WS_KO.getTableName()); + public TableToRetrieve getAClientTableByType(ExportTrackingType type) throws ArcException { + // return data table found in track table for the given type - ArcPreparedStatementBuilder requete = new ArcPreparedStatementBuilder(); - requete.append("SELECT schemaname||'.'||tablename FROM pg_tables") - .append(" WHERE tablename like " + requete.quoteText(tableToFind)) - .append(" AND schemaname=" + requete.quoteText(schema)).append(" AND tablename " - + (isSourceListTable ? "" : "NOT") + " like " + requete.quoteText("%id\\_source%")) - // ws_info must be first if exists as this table wil be always created - // others might be pending in creation process - .append(" ORDER BY CASE schemaname||'.'||tablename ") - .append(" WHEN "+requete.quoteText(tableWsInfo)+" THEN 1 ") - .append(" WHEN "+requete.quoteText(tableWsKO)+" THEN 2 ") - .append(" ELSE 3 END ") - .append(" LIMIT 1 "); - - String selectedTableName = UtilitaireDao.get(0).getString(connection, requete); - - // if selectedTableName is ws_ko, there was a problem return exception - if (selectedTableName!=null && selectedTableName.equals(tableWsKO)) - { - throw new ArcException(ArcExceptionMessage.WS_RETRIEVE_DATA_FAMILY_CREATION_FAILED); - } - - return selectedTableName==null? null : this.client + selectedTableName.substring(this.client.length()); - - } + ArcPreparedStatementBuilder query = new ArcPreparedStatementBuilder(); + query.build(SQL.SELECT, "nod, table_to_retrieve", SQL.FROM, this.tableWsTracking); + query.build(SQL.WHERE, "tracking_type=", query.quoteText(type.toString())); + query.build(SQL.LIMIT, "1"); + + Map> content = new GenericBean(UtilitaireDao.get(0).executeRequest(connection, query)) + .mapContent(); + + return content.isEmpty() ? new TableToRetrieve() + : new TableToRetrieve(content.get("nod").get(0), content.get("table_to_retrieve").get(0)); - public String getAClientTable() throws ArcException { - return getAClientTable(false); } - public String getIdTable() throws ArcException { - return getAClientTable(true); + /** + * Get the table object of the table to retrieve by its name + * + * @param tableName + * @return + * @throws ArcException + */ + public TableToRetrieve getAClientTableByName(String tableName) throws ArcException { + + ArcPreparedStatementBuilder query = new ArcPreparedStatementBuilder(); + query.build(SQL.SELECT, "nod, table_to_retrieve", SQL.FROM, this.tableWsTracking); + query.build(SQL.WHERE, "table_to_retrieve=", query.quoteText(tableName)); + query.build(SQL.LIMIT, "1"); + + Map> content = new GenericBean(UtilitaireDao.get(0).executeRequest(connection, query)) + .mapContent(); + + return content.isEmpty() ? new TableToRetrieve() + : new TableToRetrieve(content.get("nod").get(0), content.get("table_to_retrieve").get(0)); } public void dropTable(String clientTable) { + dropTable(ArcDatabase.COORDINATOR.getIndex(), clientTable); + } + + public void dropTable(int connectionIndex, String clientTable) { if (StringUtils.isBlank(clientTable)) { return; } - UtilitaireDao.get(0).dropTable(connection, clientTable); + UtilitaireDao.get(connectionIndex).dropTable(connection, clientTable); + } + + public void dropTable(TableToRetrieve table) { + + if (table.getNod().equals(ArcDatabase.EXECUTOR)) { + int numberOfExecutorNods = ArcDatabase.numberOfExecutorNods(); + for (int executorConnectionId = ArcDatabase.EXECUTOR.getIndex(); executorConnectionId < ArcDatabase.EXECUTOR + .getIndex() + numberOfExecutorNods; executorConnectionId++) { + dropTable(executorConnectionId, table.getTableName()); + } + } else { + dropTable(0, table.getTableName()); + } + } /** @@ -422,7 +460,7 @@ public void dropTable(String clientTable) { * * @throws ArcException */ - public void dropPendingClientTables() throws ArcException { + public void dropPendingClientTables(int connectionId) throws ArcException { String findClientTable = ViewEnum.normalizeTableName(client + "\\_%"); @@ -431,74 +469,87 @@ public void dropPendingClientTables() throws ArcException { requete.append(" WHERE tablename like " + requete.quoteText(findClientTable)); requete.append(" AND schemaname = " + requete.quoteText(this.environnement)); - List tablesToDrop = new GenericBean(UtilitaireDao.get(0).executeRequest(connection, requete)) + List tablesToDrop = new GenericBean(UtilitaireDao.get(connectionId).executeRequest(connection, requete)) .getColumnValues(ColumnEnum.TABLE_NAME.getColumnName()); - UtilitaireDao.get(0).executeImmediate(connection, FormatSQL.dropTable(tablesToDrop.toArray(new String[0]))); - + UtilitaireDao.get(connectionId).executeImmediate(null, + FormatSQL.dropTable(tablesToDrop.toArray(new String[0]))); } /** * create reporting table + * * @throws ArcException */ - public void createTableWsStatus() throws ArcException { + public void createTableWsInfo() throws ArcException { + + String tableWsInfo = TableNaming.buildTableNameWithTokens(environnement, ViewEnum.WS_INFO, client, timestamp); - String tableWsInfo = TableNaming.buildTableNameWithTokens(environnement, - ViewEnum.WS_INFO, client, timestamp); - ArcPreparedStatementBuilder requete = new ArcPreparedStatementBuilder(); - requete.append("\n DROP TABLE IF EXISTS "+ tableWsInfo +";"); - + requete.append("\n DROP TABLE IF EXISTS " + tableWsInfo + ";"); + requete.append("\n CREATE TABLE " + tableWsInfo + FormatSQL.WITH_NO_VACUUM + " AS"); - requete.append("\n SELECT "+ requete.quoteText(client)+ " as client "); - requete.append(", "+requete.quoteText(Long.toString(timestamp)) +" as timestamp "); + requete.append("\n SELECT " + requete.quoteText(client) + " as client "); + requete.append(", " + requete.quoteText(Long.toString(timestamp)) + " as timestamp "); requete.append(";"); - - requete.append("\n DROP TABLE IF EXISTS "+ tableWsPending +";"); + + requete.append("\n DROP TABLE IF EXISTS " + tableWsPending + ";"); requete.append("\n CREATE TABLE " + tableWsPending + "();"); - + UtilitaireDao.get(0).executeImmediate(connection, requete); + registerTableToBeRetrieved(ExportTrackingType.DATA, ArcDatabase.COORDINATOR, tableWsInfo); + } public void createTableWsKO() throws ArcException { - String tableWsKO = TableNaming.buildTableNameWithTokens(environnement, - ViewEnum.WS_KO, client, timestamp); - - ArcPreparedStatementBuilder requete = new ArcPreparedStatementBuilder(); - requete.append("\n DROP TABLE IF EXISTS "+ tableWsKO +";"); - requete.append("\n CREATE TABLE " + tableWsKO + "();"); - - UtilitaireDao.get(0).executeImmediate(connection, requete); + registerTableToBeRetrieved(ExportTrackingType.KO, ArcDatabase.COORDINATOR, ViewEnum.WS_KO.toString()); } - + public void dropTableWsPending() throws ArcException { ArcPreparedStatementBuilder requete = new ArcPreparedStatementBuilder(); - requete.append("DROP TABLE IF EXISTS "+ tableWsPending +";"); + requete.append("DROP TABLE IF EXISTS " + tableWsPending + ";"); UtilitaireDao.get(0).executeImmediate(connection, requete); } /** - * web service data creation is not pending if tableWsPending doesn't exists anymore + * web service data creation is not pending if tableWsPending doesn't exists + * anymore + * * @return * @throws ArcException */ public boolean isWebServiceNotPending() throws ArcException { + ArcPreparedStatementBuilder requete = new ArcPreparedStatementBuilder(); - - String[] schemaAndClient = this.client.split("\\"+Delimiters.SQL_SCHEMA_DELIMITER); - String schema = schemaAndClient[0]; - String[] clientTokens = schemaAndClient[1].split(Delimiters.SQL_TOKEN_DELIMITER); - String clientExtract = clientTokens[0]; - String timestampExtract = clientTokens[1]; - - String tableWsPendingExtract = TableNaming.buildTableNameWithTokens(schema, ViewEnum.WS_PENDING, clientExtract, timestampExtract); - - requete.append("SELECT 1 FROM pg_tables"); - requete.append(" WHERE schemaname||'.'||tablename = "+requete.quoteText(tableWsPendingExtract)+" "); + requete.append("SELECT 1 FROM pg_tables WHERE schemaname||'.'||tablename = " + requete.quoteText(tableWsPending) + + " "); return !UtilitaireDao.get(0).hasResults(connection, requete); } + public void copyTableOfIdSourceToExecutorNod(int connectionId) throws ArcException { + GenericBean gb = new GenericBean(UtilitaireDao.get(0).executeRequest(connection, + new ArcPreparedStatementBuilder("SELECT * FROM " + tableOfIdSource))); + + try (Connection executorConnection = UtilitaireDao.get(connectionId).getDriverConnexion()) { + CopyObjectsToDatabase.execCopyFromGenericBean(executorConnection, tableOfIdSource, gb); + } catch (SQLException e) { + ArcException customException = new ArcException(e, ArcExceptionMessage.DATABASE_CONNECTION_EXECUTOR_FAILED); + customException.logFullException(); + throw customException; + } + } + + public void deleteFromTrackTable(String tableName) throws ArcException { + ArcPreparedStatementBuilder query = new ArcPreparedStatementBuilder(); + query.build(SQL.DELETE, this.tableWsTracking); + query.build(SQL.WHERE, "table_to_retrieve=", query.quoteText(tableName)); + UtilitaireDao.get(0).executeImmediate(connection, query); + } + + public void setConnection(Connection connection) { + this.connection = connection; + } + } diff --git a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/dao/NameDao.java b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/dao/NameDao.java index cd24a63d5..e40356cf5 100644 --- a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/dao/NameDao.java +++ b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/dao/NameDao.java @@ -5,13 +5,14 @@ import fr.insee.arc.core.dataobjects.ArcPreparedStatementBuilder; import fr.insee.arc.utils.dao.UtilitaireDao; import fr.insee.arc.utils.exception.ArcException; +import fr.insee.arc.ws.services.importServlet.bo.TableToRetrieve; public class NameDao { - public static List> execQuerySelectMetadata(String tableName) throws ArcException + public static List> execQuerySelectMetadata(TableToRetrieve table) throws ArcException { - return UtilitaireDao.get(0).executeRequest(null,new ArcPreparedStatementBuilder("select * from " + tableName + " where false ")); + return UtilitaireDao.get(table.getNod().getIndex()).executeRequest(null,new ArcPreparedStatementBuilder("select * from " + table.getTableName() + " where false ")); } diff --git a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/dao/ServiceDao.java b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/dao/ServiceDao.java index 41d45b3b0..62ce0bb68 100644 --- a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/dao/ServiceDao.java +++ b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/dao/ServiceDao.java @@ -4,26 +4,43 @@ import java.io.OutputStream; import java.util.zip.GZIPOutputStream; +import fr.insee.arc.core.dataobjects.ArcDatabase; import fr.insee.arc.utils.dao.UtilitaireDao; import fr.insee.arc.utils.exception.ArcException; import fr.insee.arc.utils.exception.ArcExceptionMessage; +import fr.insee.arc.ws.services.importServlet.bo.TableToRetrieve; public class ServiceDao { - public static void execQueryExportDataToResponse(OutputStream os, String tableName, boolean csvExportFormat) throws ArcException { + public static void execQueryExportDataToResponse(OutputStream os, TableToRetrieve table, boolean csvExportFormat) throws ArcException { if (csvExportFormat) { try(GZIPOutputStream goz=new GZIPOutputStream(os);) { - UtilitaireDao.get(0).exporting(null, tableName, goz, csvExportFormat); + if (table.getNod().equals(ArcDatabase.EXECUTOR)) + { + int numberOfExecutorNods = ArcDatabase.numberOfExecutorNods(); + for (int executorConnectionId = ArcDatabase.EXECUTOR.getIndex(); executorConnectionId < ArcDatabase.EXECUTOR + .getIndex() + numberOfExecutorNods; executorConnectionId++) { + UtilitaireDao.get(executorConnectionId).exporting(null, table.getTableName(), goz, csvExportFormat); + } + } + else + { + UtilitaireDao.get(0).exporting(null, table.getTableName(), goz, csvExportFormat); + } } catch (IOException e) { throw new ArcException(ArcExceptionMessage.STREAM_WRITE_FAILED); } } else { - UtilitaireDao.get(0).exporting(null, tableName, os, csvExportFormat); + if (table.getNod().equals(ArcDatabase.EXECUTOR)) + { + throw new ArcException(ArcExceptionMessage.WS_RETRIEVE_DATA_SCALABLE_TABLE_MUST_BE_EXPORT_IN_CSV); + } + UtilitaireDao.get(0).exporting(null, table.getTableName(), os, csvExportFormat); } } diff --git a/arc-ws/src/test/java/fr/insee/arc/ws/services/importServlet/bo/ArcClientIdentifierTest.java b/arc-ws/src/test/java/fr/insee/arc/ws/services/importServlet/bo/ArcClientIdentifierTest.java new file mode 100644 index 000000000..54ada3262 --- /dev/null +++ b/arc-ws/src/test/java/fr/insee/arc/ws/services/importServlet/bo/ArcClientIdentifierTest.java @@ -0,0 +1,40 @@ +package fr.insee.arc.ws.services.importServlet.bo; + +import static org.junit.Assert.*; + +import org.json.JSONObject; +import org.junit.Test; + +public class ArcClientIdentifierTest { + + @Test + public void testArcClientIdentifierInitializeClient() { + + JSONObject json = new JSONObject("{\"client\":\"ARTEMIS\",\"environnement\":\"arc.bas1\",\"familleNorme\":\"DSN\",\"format\":\"csv_gzip\"}"); + ArcClientIdentifier clientParameters = new ArcClientIdentifier(json, true); + + assertEquals("ARTEMIS",clientParameters.getClientInputParameter()); + assertEquals("ARTEMIS",clientParameters.getClientIdentifier()); + assertEquals("arc_bas1",clientParameters.getEnvironnement()); + assertEquals("DSN",clientParameters.getFamille()); + assertEquals(ExportFormat.CSV_GZIP.getFormat(),clientParameters.getFormat()); + } + + + @Test + public void testArcClientIdentifierRetrieveClientAttributes() { + + JSONObject json = new JSONObject("{\"client\":\"arc_bas1.ARTEMIS_1701335653112_nmcl_code_pays_etranger_2015\",\"environnement\":\"arc.bas1\",\"familleNorme\":\"DSN\",\"format\":\"csv_gzip\"}"); + + ArcClientIdentifier clientParameters = new ArcClientIdentifier(json, false); + + assertEquals("arc_bas1.ARTEMIS_1701335653112_nmcl_code_pays_etranger_2015",clientParameters.getClientInputParameter()); + assertEquals("ARTEMIS",clientParameters.getClientIdentifier()); + assertEquals(1701335653112L,clientParameters.getTimestamp()); + assertEquals("arc_bas1",clientParameters.getEnvironnement()); + assertEquals("DSN",clientParameters.getFamille()); + assertEquals(ExportFormat.CSV_GZIP.getFormat(),clientParameters.getFormat()); + + } + +} diff --git a/arc-ws/src/test/java/fr/insee/arc/ws/services/importServlet/dao/ClientDaoTest.java b/arc-ws/src/test/java/fr/insee/arc/ws/services/importServlet/dao/ClientDaoTest.java new file mode 100644 index 000000000..50aea94ab --- /dev/null +++ b/arc-ws/src/test/java/fr/insee/arc/ws/services/importServlet/dao/ClientDaoTest.java @@ -0,0 +1,103 @@ +package fr.insee.arc.ws.services.importServlet.dao; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.sql.SQLException; +import java.util.List; + +import org.json.JSONObject; +import org.junit.Test; + +import fr.insee.arc.core.dataobjects.ArcPreparedStatementBuilder; +import fr.insee.arc.utils.dao.SQL; +import fr.insee.arc.utils.dao.UtilitaireDao; +import fr.insee.arc.utils.exception.ArcException; +import fr.insee.arc.utils.query.InitializeQueryTest; +import fr.insee.arc.utils.ressourceUtils.PropertiesHandler; +import fr.insee.arc.ws.services.importServlet.bo.ArcClientIdentifier; + +public class ClientDaoTest extends InitializeQueryTest { + + @Test + public void clientDaoTest() throws ArcException, SQLException { + + InitializeQueryTest.buildPropertiesWithoutScalability("any"); + + initializeTestData(); + + testVerificationFamilleOK(); + testVerificationFamilleKO(); + + testSelectBusinessDataTables(); + + destroyTestData(); + } + + private void testSelectBusinessDataTables() throws ArcException { + JSONObject json = new JSONObject( + "{\"client\":\"ARTEMIS\",\"environnement\":\"arc.bas1\",\"familleNorme\":\"DSN\",\"format\":\"csv_gzip\"}"); + ArcClientIdentifier queryParameters = new ArcClientIdentifier(json, true); + ClientDao clientDao = new ClientDao(queryParameters); + List clientTables = clientDao.selectBusinessDataTables(); + + assertTrue(clientTables.contains("mapping_dsn_test1_ok")); + assertTrue(clientTables.contains("mapping_dsn_test2_ok")); + assertEquals(2,clientTables.size()); + } + + @Test + public void testVerificationFamilleOK() throws ArcException { + JSONObject json = new JSONObject( + "{\"client\":\"ARTEMIS\",\"environnement\":\"arc.bas1\",\"familleNorme\":\"DSN\",\"format\":\"csv_gzip\"}"); + ArcClientIdentifier queryParameters = new ArcClientIdentifier(json, true); + ClientDao clientDao = new ClientDao(queryParameters); + assertTrue(clientDao.verificationClientFamille()); + } + + @Test + public void testVerificationFamilleKO() throws ArcException { + JSONObject json = new JSONObject( + "{\"client\":\"ARTEMIS\",\"environnement\":\"arc.bas1\",\"familleNorme\":\"BATI\",\"format\":\"csv_gzip\"}"); + ArcClientIdentifier queryParameters = new ArcClientIdentifier(json, true); + ClientDao clientDao = new ClientDao(queryParameters); + assertFalse(clientDao.verificationClientFamille()); + } + + + private void initializeTestData() throws SQLException, ArcException { + + ArcPreparedStatementBuilder query; + + query = new ArcPreparedStatementBuilder(); + + query.append("CREATE SCHEMA arc;"); + query.append("CREATE SCHEMA arc_bas1;"); + + query.append("CREATE TABLE arc.ihm_client AS "); + query.append("SELECT 'DSN' as id_famille,'ARTEMIS' as id_application UNION ALL "); + query.append("SELECT 'DSN' as id_famille,'DSNFLASH' as id_application"); + query.append(SQL.END_QUERY); + + query.append("CREATE TABLE arc_bas1.ihm_mod_table_metier AS "); + query.append("SELECT 'DSN' as id_famille,'mapping_dsn_test1_ok' as nom_table_metier UNION ALL "); + query.append("SELECT 'DSN' as id_famille,'mapping_dsn_test2_ok' as nom_table_metier UNION ALL "); + query.append("SELECT 'PASRAU' as id_famille,'mapping_pasrau_test_ok' as nom_table_metier"); + query.append(SQL.END_QUERY); + + UtilitaireDao.get(0).executeImmediate(c, query); + } + + private void destroyTestData() throws SQLException, ArcException { + + ArcPreparedStatementBuilder query; + + query = new ArcPreparedStatementBuilder(); + + query.append("DROP SCHEMA arc CASCADE;"); + query.append("DROP SCHEMA arc_bas1 CASCADE;"); + UtilitaireDao.get(0).executeImmediate(c, query); + } + +}