From f8f8898e7105c06929e37c9e5922fc5ee9660554 Mon Sep 17 00:00:00 2001 From: Manuel Soulier Date: Fri, 1 Dec 2023 17:46:51 +0100 Subject: [PATCH] feat: test on data table image creation --- .../services/importServlet/dao/ClientDao.java | 12 +- .../importServlet/dao/ClientDaoTest.java | 108 ++++++++++++++---- 2 files changed, 96 insertions(+), 24 deletions(-) diff --git a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/dao/ClientDao.java b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/dao/ClientDao.java index 804054b2e..96e8966f8 100644 --- a/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/dao/ClientDao.java +++ b/arc-ws/src/main/java/fr/insee/arc/ws/services/importServlet/dao/ClientDao.java @@ -2,6 +2,7 @@ import java.sql.Connection; import java.sql.SQLException; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -131,7 +132,7 @@ private void registerTableToBeRetrieved(ExportTrackingType wsTrackingType, ArcDa * @return liste des noms de tables images crées * @throws ArcException */ - private void addImage(String tableMetier, int executorConnectionId) throws ArcException { + private String addImage(String tableMetier, int executorConnectionId) throws ArcException { StringBuilder request = new StringBuilder(); String nomTableImage = TableNaming.buildTableNameWithTokens(environnement, tableMetier, client, timestamp); @@ -147,6 +148,8 @@ private void addImage(String tableMetier, int executorConnectionId) throws ArcEx UtilitaireDao.get(executorConnectionId).executeBlock(connection, request); registerTableToBeRetrieved(ExportTrackingType.DATA, ArcDatabase.EXECUTOR, nomTableImage); + + return nomTableImage; } @@ -264,12 +267,15 @@ public void createTableOfIdSource(JSONObject requeteJSON) throws ArcException { * @return liste des noms de tables images crées * @throws ArcException */ - public void createImages(List tablesMetierNames, int executorConnectionId) throws ArcException { + public List createImages(List tablesMetierNames, int executorConnectionId) throws ArcException { LoggerHelper.debugAsComment(LOGGER, timestamp, "ClientDaoImpl.createImage()"); + List dataTableImages = new ArrayList<>(); + for (String tableMetier : tablesMetierNames) { - addImage(tableMetier, executorConnectionId); + dataTableImages.add(addImage(tableMetier, executorConnectionId)); } + return dataTableImages; } /* diff --git a/arc-ws/src/test/java/fr/insee/arc/ws/services/importServlet/dao/ClientDaoTest.java b/arc-ws/src/test/java/fr/insee/arc/ws/services/importServlet/dao/ClientDaoTest.java index ed1fbbf93..a28a37f41 100644 --- a/arc-ws/src/test/java/fr/insee/arc/ws/services/importServlet/dao/ClientDaoTest.java +++ b/arc-ws/src/test/java/fr/insee/arc/ws/services/importServlet/dao/ClientDaoTest.java @@ -10,6 +10,7 @@ import org.json.JSONObject; import org.junit.Test; +import fr.insee.arc.core.dataobjects.ArcDatabase; import fr.insee.arc.core.dataobjects.ArcPreparedStatementBuilder; import fr.insee.arc.utils.dao.SQL; import fr.insee.arc.utils.dao.UtilitaireDao; @@ -18,12 +19,13 @@ import fr.insee.arc.utils.structure.GenericBean; import fr.insee.arc.ws.services.importServlet.bo.ArcClientIdentifier; import fr.insee.arc.ws.services.importServlet.bo.ExportTrackingType; +import fr.insee.arc.ws.services.importServlet.bo.TableToRetrieve; public class ClientDaoTest extends InitializeQueryTest { - // request for DSN family, ARTEMIS client and reprise = true + // request for DSN family, ARTEMIS client and reprise = false JSONObject jsonDsnStep1 = new JSONObject( - "{\"familleNorme\":\"DSN\",\"periodicite\":\"M\",\"service\":\"arcClient\",\"validiteSup\":\"2032-03-01\",\"format\":\"csv_gzip\",\"reprise\":true,\"client\":\"ARTEMIS\",\"environnement\":\"arc_bas1\"}"); + "{\"familleNorme\":\"DSN\",\"periodicite\":\"M\",\"service\":\"arcClient\",\"validiteSup\":\"2032-03-01\",\"format\":\"csv_gzip\",\"reprise\":false,\"client\":\"ARTEMIS\",\"environnement\":\"arc_bas1\"}"); ArcClientIdentifier queryParametersDsnStep1 = new ArcClientIdentifier(jsonDsnStep1, true); ClientDao clientDaoDsnStep1 = new ClientDao(queryParametersDsnStep1); @@ -44,44 +46,104 @@ public void clientDaoTest() throws ArcException, SQLException { testVerificationFamilleKO(); // test data tables retrieved according to query - testSelectBusinessDataTables(); + List selectedDataTables = testSelectBusinessDataTables(); + // test id_source selection table testCreateTableOfIdSourceRepriseFalse(); testCreateTableOfIdSourceRepriseTrue(); + // test data table image creation + // table must had been registered in track table + List dataTableImages = testCreateImages(selectedDataTables); + + // test return table from track table + // the dataTable in dataTableImages must be found the the track data table with type ExportTrackingType.DATA + testGetAClientTableByType(dataTableImages); + // the dataTable in dataTableImages must be found the the track data table by its name + testGetAClientTableByName(dataTableImages); + destroyTestData(); } - private void testCreateTableOfIdSourceRepriseTrue() throws ArcException { + private void testGetAClientTableByType(List dataTableImages) throws ArcException { + TableToRetrieve registeredTable = clientDaoDsnStep1.getAClientTableByType(ExportTrackingType.DATA); + + // now that image had been created we should find it in tracking table + // check the name + assertEquals(dataTableImages.get(0),registeredTable.getTableName()); + // data table are found on executor nod + assertEquals(ArcDatabase.EXECUTOR,registeredTable.getNod()); + } + + private void testGetAClientTableByName(List dataTableImages) throws ArcException { + + TableToRetrieve registeredTable = clientDaoDsnStep1.getAClientTableByName(dataTableImages.get(0)); + + // now that image had been created we should find it in tracking table + // check the name + assertEquals(dataTableImages.get(0),registeredTable.getTableName()); + // the test is in non scalable nod so the data table must be on coordinator + assertEquals(ArcDatabase.EXECUTOR,registeredTable.getNod()); + } + + private List testCreateImages(List selectedDataTables) throws ArcException { + List dataTableImages = clientDaoDsnStep1.createImages(selectedDataTables, 0); + + // only 1 table in model and 1 table should had been created + assertEquals(1, dataTableImages.size()); + + ArcPreparedStatementBuilder query = new ArcPreparedStatementBuilder(); + query.append("SELECT distinct id_source FROM "+dataTableImages.get(0)+";"); + List content = new GenericBean(UtilitaireDao.get(0).executeRequest(c, query)).getColumnValues("id_source"); + + // only table with 1 id_source must had been retrieved + assertEquals(1, content.size()); + + return dataTableImages; + + } + + /** + * test on retrieving idSource + * request on DSN family, ARTEMIS client and reprise = false + * as reprise = false, only files not already retrieved by client must be selected + * @throws ArcException + */ + private void testCreateTableOfIdSourceRepriseFalse() throws ArcException { + clientDaoDsnStep1.createTableOfIdSource(jsonDsnStep1); ArcPreparedStatementBuilder query = new ArcPreparedStatementBuilder(); query.append("SELECT id_source FROM "+clientDaoDsnStep1.getTableOfIdSource()+";"); - List content = new GenericBean(UtilitaireDao.get(0).executeRequest(c, query)).getColumnValues("id_source"); - assertEquals(2, content.size()); + + // only 1 file must be selected as reprise = false + // file_not_to_retrieve_when_reprise_false has already been marked as retrieved by 'ARTEMIS' client + assertEquals(1, content.size()); } - private void testCreateTableOfIdSourceRepriseFalse() throws ArcException { + /** + * test to select id_source to be retrieved when reprise=true + * @throws ArcException + */ + private void testCreateTableOfIdSourceRepriseTrue() throws ArcException { - // request on DSN family, ARTEMIS client and reprise = false - // as reprise = false, only files not already retrieved by client must be selected - JSONObject jsonDsnStep1RepriseFalse = new JSONObject( - "{\"familleNorme\":\"DSN\",\"periodicite\":\"M\",\"service\":\"arcClient\",\"validiteSup\":\"2032-03-01\",\"format\":\"csv_gzip\",\"reprise\":false,\"client\":\"ARTEMIS\",\"environnement\":\"arc_bas1\"}"); - ArcClientIdentifier queryParametersDsnStep1RepriseFalse = new ArcClientIdentifier(jsonDsnStep1RepriseFalse, true); - ClientDao clientDaoDsnStep1RepriseFalse = new ClientDao(queryParametersDsnStep1RepriseFalse); + JSONObject jsonDsnStep1RepriseTrue = new JSONObject( + "{\"familleNorme\":\"DSN\",\"periodicite\":\"M\",\"service\":\"arcClient\",\"validiteSup\":\"2032-03-01\",\"format\":\"csv_gzip\",\"reprise\":true,\"client\":\"ARTEMIS\",\"environnement\":\"arc_bas1\"}"); + ArcClientIdentifier queryParametersDsnStep1RepriseTrue = new ArcClientIdentifier(jsonDsnStep1RepriseTrue, true); + ClientDao clientDaoDsnStep1RepriseTrue = new ClientDao(queryParametersDsnStep1RepriseTrue); // create tracking table - clientDaoDsnStep1RepriseFalse.createTableTrackRetrievedTables(); + clientDaoDsnStep1RepriseTrue.createTableTrackRetrievedTables(); - clientDaoDsnStep1RepriseFalse.createTableOfIdSource(jsonDsnStep1RepriseFalse); + clientDaoDsnStep1RepriseTrue.createTableOfIdSource(jsonDsnStep1RepriseTrue); ArcPreparedStatementBuilder query = new ArcPreparedStatementBuilder(); - query.append("SELECT id_source FROM "+clientDaoDsnStep1RepriseFalse.getTableOfIdSource()+";"); + query.append("SELECT id_source FROM "+clientDaoDsnStep1RepriseTrue.getTableOfIdSource()+";"); List content = new GenericBean(UtilitaireDao.get(0).executeRequest(c, query)).getColumnValues("id_source"); // only 1 file must be selected as reprise = false // file_not_to_retrieve_when_reprise_false has already been marked as retrieved by 'ARTEMIS' client - assertEquals(1, content.size()); + assertEquals(2, content.size()); } private void testCreateTableTrackRetrievedTables() throws ArcException { @@ -99,13 +161,13 @@ private void testCreateTableTrackRetrievedTables() throws ArcException { } - private void testSelectBusinessDataTables() throws ArcException { + private List testSelectBusinessDataTables() throws ArcException { List clientTables = clientDaoDsnStep1.selectBusinessDataTables(); assertTrue(clientTables.contains("mapping_dsn_test1_ok")); - assertTrue(clientTables.contains("mapping_dsn_test2_ok")); - assertEquals(2,clientTables.size()); + assertEquals(1,clientTables.size()); + return clientTables; } public void testVerificationFamilleOK() throws ArcException { @@ -146,7 +208,6 @@ private void initializeTestData() throws SQLException, ArcException { query.append("CREATE TABLE arc_bas1.mod_table_metier AS "); query.append("SELECT 'DSN' as id_famille,'mapping_dsn_test1_ok' as nom_table_metier UNION ALL "); - query.append("SELECT 'DSN' as id_famille,'mapping_dsn_test2_ok' as nom_table_metier UNION ALL "); query.append("SELECT 'PASRAU' as id_famille,'mapping_pasrau_test_ok' as nom_table_metier"); query.append(SQL.END_QUERY); @@ -165,6 +226,11 @@ private void initializeTestData() throws SQLException, ArcException { query.append("SELECT 'PHASE3V1' as id_norme, 'DSN' as id_famille UNION ALL "); query.append("SELECT 'PASRAU' as id_norme, 'PASRAU' as id_famille"); query.append(SQL.END_QUERY); + + query.append("CREATE TABLE arc_bas1.mapping_dsn_test1_ok AS "); + query.append("SELECT 'file_to_retrieve.xml' as id_source, 'data_of_file_to_retrieve' as data UNION ALL "); + query.append("SELECT 'file_not_to_retrieve_when_reprise_false.xml' as id_source, 'data_of_file_not_to_retrieve_when_reprise_false' as data"); + query.append(SQL.END_QUERY); UtilitaireDao.get(0).executeImmediate(c, query); }