diff --git a/arc-batch/src/main/java/fr/insee/arc/batch/threadrunners/PhaseThreadFactory.java b/arc-batch/src/main/java/fr/insee/arc/batch/threadrunners/PhaseThreadFactory.java index 6eaf56852..516157165 100644 --- a/arc-batch/src/main/java/fr/insee/arc/batch/threadrunners/PhaseThreadFactory.java +++ b/arc-batch/src/main/java/fr/insee/arc/batch/threadrunners/PhaseThreadFactory.java @@ -30,7 +30,6 @@ public void execute() { this.report = ApiServiceFactory.getService( // getPhaseName(), // mapParam.get(PhaseParameterKeys.KEY_FOR_EXECUTION_ENVIRONMENT), // - mapParam.get(PhaseParameterKeys.KEY_FOR_DIRECTORY_LOCATION), // Integer.parseInt(mapParam.get(capacityParameterName())), // Boolean.parseBoolean(PhaseParameterKeys.KEY_FOR_KEEP_IN_DATABASE) ? null : mapParam.get(PhaseParameterKeys.KEY_FOR_BATCH_CHUNK_ID)) // diff --git a/arc-core/src/main/java/fr/insee/arc/core/factory/ApiChargementServiceFactory.java b/arc-core/src/main/java/fr/insee/arc/core/factory/ApiChargementServiceFactory.java index 75446c468..4966f7533 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/factory/ApiChargementServiceFactory.java +++ b/arc-core/src/main/java/fr/insee/arc/core/factory/ApiChargementServiceFactory.java @@ -15,8 +15,8 @@ public class ApiChargementServiceFactory implements IServiceFactory { * @param aDirectoryRoot * @param aNbEnr */ - public ApiService get(TraitementPhase phaseService, String executionSchema, String directory, Integer capacityParameter, String paramBatch) { - return new ApiChargementService(phaseService, executionSchema, directory, capacityParameter, paramBatch); + public ApiService get(TraitementPhase phaseService, String executionSchema, Integer capacityParameter, String paramBatch) { + return new ApiChargementService(phaseService, executionSchema, capacityParameter, paramBatch); } public static IServiceFactory getInstance() { diff --git a/arc-core/src/main/java/fr/insee/arc/core/factory/ApiControleServiceFactory.java b/arc-core/src/main/java/fr/insee/arc/core/factory/ApiControleServiceFactory.java index 07f34c3df..3a75d497f 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/factory/ApiControleServiceFactory.java +++ b/arc-core/src/main/java/fr/insee/arc/core/factory/ApiControleServiceFactory.java @@ -15,8 +15,8 @@ public class ApiControleServiceFactory implements IServiceFactory { * @param aDirectoryRoot * @param aNbEnr */ - public ApiService get(TraitementPhase phaseService, String executionSchema, String directory, Integer capacityParameter, String paramBatch) { - return new ApiControleService(phaseService, executionSchema, directory, capacityParameter, paramBatch); + public ApiService get(TraitementPhase phaseService, String executionSchema, Integer capacityParameter, String paramBatch) { + return new ApiControleService(phaseService, executionSchema, capacityParameter, paramBatch); } public static IServiceFactory getInstance() { diff --git a/arc-core/src/main/java/fr/insee/arc/core/factory/ApiInitialisationServiceFactory.java b/arc-core/src/main/java/fr/insee/arc/core/factory/ApiInitialisationServiceFactory.java index 05f79cb99..382599f3f 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/factory/ApiInitialisationServiceFactory.java +++ b/arc-core/src/main/java/fr/insee/arc/core/factory/ApiInitialisationServiceFactory.java @@ -7,8 +7,8 @@ public class ApiInitialisationServiceFactory implements IServiceFactory { @Override - public ApiService get(TraitementPhase phaseService, String executionSchema, String directory, Integer capacityParameter, String paramBatch) { - return new ApiInitialisationService(phaseService, executionSchema, directory, capacityParameter, paramBatch); + public ApiService get(TraitementPhase phaseService, String executionSchema, Integer capacityParameter, String paramBatch) { + return new ApiInitialisationService(phaseService, executionSchema, capacityParameter, paramBatch); } public static IServiceFactory getInstance() { diff --git a/arc-core/src/main/java/fr/insee/arc/core/factory/ApiMappingServiceFactory.java b/arc-core/src/main/java/fr/insee/arc/core/factory/ApiMappingServiceFactory.java index 104f2c990..e79a615e5 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/factory/ApiMappingServiceFactory.java +++ b/arc-core/src/main/java/fr/insee/arc/core/factory/ApiMappingServiceFactory.java @@ -7,8 +7,8 @@ public class ApiMappingServiceFactory implements IServiceFactory { @Override - public ApiService get(TraitementPhase phaseService, String executionSchema, String directory, Integer capacityParameter, String paramBatch) { - return new ApiMappingService(phaseService, executionSchema, directory, capacityParameter, paramBatch); + public ApiService get(TraitementPhase phaseService, String executionSchema, Integer capacityParameter, String paramBatch) { + return new ApiMappingService(phaseService, executionSchema, capacityParameter, paramBatch); } public static IServiceFactory getInstance() { diff --git a/arc-core/src/main/java/fr/insee/arc/core/factory/ApiNormageServiceFactory.java b/arc-core/src/main/java/fr/insee/arc/core/factory/ApiNormageServiceFactory.java index a3107bcfc..270c43cb2 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/factory/ApiNormageServiceFactory.java +++ b/arc-core/src/main/java/fr/insee/arc/core/factory/ApiNormageServiceFactory.java @@ -15,8 +15,8 @@ public class ApiNormageServiceFactory implements IServiceFactory { * @param aDirectoryRoot * @param aNbEnr */ - public ApiService get(TraitementPhase phaseService, String executionSchema, String directory, Integer capacityParameter, String paramBatch) { - return new ApiNormageService(phaseService, executionSchema, directory, capacityParameter, paramBatch); + public ApiService get(TraitementPhase phaseService, String executionSchema, Integer capacityParameter, String paramBatch) { + return new ApiNormageService(phaseService, executionSchema, capacityParameter, paramBatch); } public static IServiceFactory getInstance() { diff --git a/arc-core/src/main/java/fr/insee/arc/core/factory/ApiReceptionServiceFactory.java b/arc-core/src/main/java/fr/insee/arc/core/factory/ApiReceptionServiceFactory.java index f609b8465..a6d65b12f 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/factory/ApiReceptionServiceFactory.java +++ b/arc-core/src/main/java/fr/insee/arc/core/factory/ApiReceptionServiceFactory.java @@ -15,8 +15,8 @@ public class ApiReceptionServiceFactory implements IServiceFactory { * @param aDirectoryRoot * @param aNbEnr */ - public ApiService get(TraitementPhase phaseService, String executionSchema, String directory, Integer capacityParameter, String paramBatch) { - return new ApiReceptionService(phaseService, executionSchema, directory, capacityParameter, paramBatch); + public ApiService get(TraitementPhase phaseService, String executionSchema, Integer capacityParameter, String paramBatch) { + return new ApiReceptionService(phaseService, executionSchema, capacityParameter, paramBatch); } public static IServiceFactory getInstance() { diff --git a/arc-core/src/main/java/fr/insee/arc/core/factory/ApiServiceFactory.java b/arc-core/src/main/java/fr/insee/arc/core/factory/ApiServiceFactory.java index 463065bc8..99020bcf7 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/factory/ApiServiceFactory.java +++ b/arc-core/src/main/java/fr/insee/arc/core/factory/ApiServiceFactory.java @@ -52,7 +52,7 @@ private static final ApiServiceFactory getInstance() { * @param paramBatch * @return */ - public static final ApiService getService(TraitementPhase phaseService, String executionSchema, String directory, Integer capacityParameter, String paramBatch) { - return getInstance().map.get(phaseService).get(phaseService, executionSchema, directory, capacityParameter, paramBatch); + public static final ApiService getService(TraitementPhase phaseService, String executionSchema, Integer capacityParameter, String paramBatch) { + return getInstance().map.get(phaseService).get(phaseService, executionSchema, capacityParameter, paramBatch); } } diff --git a/arc-core/src/main/java/fr/insee/arc/core/factory/IServiceFactory.java b/arc-core/src/main/java/fr/insee/arc/core/factory/IServiceFactory.java index 760fb5d91..b3f4df760 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/factory/IServiceFactory.java +++ b/arc-core/src/main/java/fr/insee/arc/core/factory/IServiceFactory.java @@ -15,6 +15,6 @@ public interface IServiceFactory { * @param paramBatch * @return */ - public ApiService get(TraitementPhase phaseService, String executionSchema, String directory, Integer capacityParameter, String paramBatch); + public ApiService get(TraitementPhase phaseService, String executionSchema, Integer capacityParameter, String paramBatch); } diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/global/ApiService.java b/arc-core/src/main/java/fr/insee/arc/core/service/global/ApiService.java index 0c807534d..89b7c9239 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/global/ApiService.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/global/ApiService.java @@ -72,7 +72,7 @@ public ApiService() { springInit(); } - protected ApiService(TraitementPhase aCurrentPhase, String aEnvExecution, String aDirectoryRoot, Integer aNbEnr, + protected ApiService(TraitementPhase aCurrentPhase, String aEnvExecution, Integer aNbEnr, String paramBatch) { StaticLoggerDispatcher.info(LOGGER_APISERVICE, "** initialiserVariable **"); diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/ApiInitialisationService.java b/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/ApiInitialisationService.java index 909189e1a..17f606a36 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/ApiInitialisationService.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/ApiInitialisationService.java @@ -31,9 +31,9 @@ public ApiInitialisationService() { super(); } - public ApiInitialisationService(TraitementPhase aCurrentPhase, String aEnvExecution, String aDirectoryRoot, Integer aNbEnr, + public ApiInitialisationService(TraitementPhase aCurrentPhase, String aEnvExecution, Integer aNbEnr, String paramBatch) { - super(aCurrentPhase, aEnvExecution, aDirectoryRoot, aNbEnr, paramBatch); + super(aCurrentPhase, aEnvExecution, aNbEnr, paramBatch); } @Override diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/ResetEnvironmentService.java b/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/ResetEnvironmentService.java index c38bd61ca..ac2c17d57 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/ResetEnvironmentService.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p0initialisation/ResetEnvironmentService.java @@ -32,8 +32,7 @@ public static void backToTargetPhase(TraitementPhase phaseAExecuter, String env, if (phaseAExecuter.getOrdre() == TraitementPhase.INITIALISATION.getOrdre()) { resetBAS(env, rootDirectory); } else { - ApiInitialisationService serv = new ApiInitialisationService(TraitementPhase.INITIALISATION, env, - rootDirectory, TraitementPhase.INITIALISATION.getNbLigneATraiter(), null); + ApiInitialisationService serv = new ApiInitialisationService(TraitementPhase.INITIALISATION, env, TraitementPhase.INITIALISATION.getNbLigneATraiter(), null); try { new ResetEnvironmentOperation(serv.getCoordinatorSandbox()).retourPhasePrecedente(phaseAExecuter, undoFilesSelection); } finally { @@ -51,8 +50,7 @@ public static void backToTargetPhase(TraitementPhase phaseAExecuter, String env, */ public static void resetBAS(String env, String rootDirectory) { - ApiInitialisationService service = new ApiInitialisationService(TraitementPhase.INITIALISATION, env, - rootDirectory, TraitementPhase.INITIALISATION.getNbLigneATraiter(), null); + ApiInitialisationService service = new ApiInitialisationService(TraitementPhase.INITIALISATION, env, TraitementPhase.INITIALISATION.getNbLigneATraiter(), null); try { // delete files and pilotage tables new ResetEnvironmentOperation(service.getCoordinatorSandbox()).clearPilotageAndDirectories(rootDirectory); diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p1reception/ApiReceptionService.java b/arc-core/src/main/java/fr/insee/arc/core/service/p1reception/ApiReceptionService.java index f7887b305..70244d3bb 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p1reception/ApiReceptionService.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p1reception/ApiReceptionService.java @@ -31,8 +31,8 @@ public ApiReceptionService() { } public ApiReceptionService(TraitementPhase aCurrentPhase, String aEnvExecution, - String aDirectoryRoot, Integer aNbEnr, String paramBatch) { - super(aCurrentPhase, aEnvExecution, aDirectoryRoot, aNbEnr, paramBatch); + Integer aNbEnr, String paramBatch) { + super(aCurrentPhase, aEnvExecution, aNbEnr, paramBatch); } @Override diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/ApiChargementService.java b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/ApiChargementService.java index 02945738b..5934fd948 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/ApiChargementService.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p2chargement/ApiChargementService.java @@ -46,9 +46,9 @@ public ApiChargementService() { super(); } - public ApiChargementService(TraitementPhase aCurrentPhase, String aEnvExecution, String aDirectoryRoot, Integer aNbEnr, + public ApiChargementService(TraitementPhase aCurrentPhase, String aEnvExecution, Integer aNbEnr, String paramBatch) { - super(aCurrentPhase, aEnvExecution, aDirectoryRoot, aNbEnr, paramBatch); + super(aCurrentPhase, aEnvExecution, aNbEnr, paramBatch); } protected List listeNorme; diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p3normage/ApiNormageService.java b/arc-core/src/main/java/fr/insee/arc/core/service/p3normage/ApiNormageService.java index e82c68874..ecb577b0b 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p3normage/ApiNormageService.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p3normage/ApiNormageService.java @@ -41,9 +41,9 @@ public ApiNormageService() { super(); } - public ApiNormageService(TraitementPhase aCurrentPhase, String aEnvExecution, String aDirectoryRoot, Integer aNbEnr, + public ApiNormageService(TraitementPhase aCurrentPhase, String aEnvExecution, Integer aNbEnr, String paramBatch) { - super(aCurrentPhase, aEnvExecution, aDirectoryRoot, aNbEnr, paramBatch); + super(aCurrentPhase, aEnvExecution, aNbEnr, paramBatch); } @Override diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p4controle/ApiControleService.java b/arc-core/src/main/java/fr/insee/arc/core/service/p4controle/ApiControleService.java index 2e03f5580..a2ebc20d5 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p4controle/ApiControleService.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p4controle/ApiControleService.java @@ -38,9 +38,9 @@ public ApiControleService() { super(); } - public ApiControleService(TraitementPhase aCurrentPhase, String aEnvExecution, String aDirectoryRoot, Integer aNbEnr, + public ApiControleService(TraitementPhase aCurrentPhase, String aEnvExecution, Integer aNbEnr, String paramBatch) { - super(aCurrentPhase, aEnvExecution, aDirectoryRoot, aNbEnr, paramBatch); + super(aCurrentPhase, aEnvExecution, aNbEnr, paramBatch); } /** diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p5mapping/ApiMappingService.java b/arc-core/src/main/java/fr/insee/arc/core/service/p5mapping/ApiMappingService.java index 9b65a6c58..69af322ce 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p5mapping/ApiMappingService.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p5mapping/ApiMappingService.java @@ -54,9 +54,9 @@ public ApiMappingService() { * @param aCurrentPhase * @param aNbEnr */ - public ApiMappingService(TraitementPhase aCurrentPhase, String anEnvironnementExecution, String aDirectoryRoot, + public ApiMappingService(TraitementPhase aCurrentPhase, String anEnvironnementExecution, Integer aNbEnr, String paramBatch) { - super(aCurrentPhase, anEnvironnementExecution, null, aNbEnr, paramBatch); + super(aCurrentPhase, anEnvironnementExecution, aNbEnr, paramBatch); } /** diff --git a/arc-core/src/main/resources/BdD/script_global.sql b/arc-core/src/main/resources/BdD/script_global.sql index 2881ef1a7..dc444faf0 100644 --- a/arc-core/src/main/resources/BdD/script_global.sql +++ b/arc-core/src/main/resources/BdD/script_global.sql @@ -95,18 +95,22 @@ INSERT INTO arc.parameter VALUES ('MappingService.MAX_PARALLEL_WORKERS','4'); UPDATE arc.parameter set description='parameter.parallel.numberOfThread.p4.mapmodel' where key='MappingService.MAX_PARALLEL_WORKERS'; -- kubernetes parameters -INSERT INTO arc.parameter VALUES ('ArcAction.enableKube','false'); -UPDATE arc.parameter set description='parameter.ihm.enableKube' where key='ArcAction.enableKube'; -INSERT INTO arc.parameter VALUES ('ArcAction.nbExecutorPods','10'); -UPDATE arc.parameter set description='parameter.ihm.nbExecutorPods' where key='ArcAction.nbExecutorPods'; +-- horizontal scalability parameters +INSERT INTO arc.parameter VALUES ('kubernetes.gui.scalability.enable','false', 'Enable kubernetes horizontal scalability in gui mode'); +INSERT INTO arc.parameter VALUES ('kubernetes.batch.scalability.enable','false', 'Enable kubernetes horizontal scalability in batch mode'); +INSERT INTO arc.parameter VALUES ('kubernetes.batch.scalability.numberOfExecutorDatabase','4', 'Number of executor database nod for horizontal scaling'); -INSERT INTO arc.parameter VALUES ('BatchArc.enableKube','false'); -UPDATE arc.parameter set description='parameter.batch.enableKube' where key='BatchArc.enableKube'; - -INSERT INTO arc.parameter VALUES ('BatchArc.nbExecutorPods','10'); -UPDATE arc.parameter set description='parameter.batch.nbExecutorPods' where key='BatchArc.nbExecutorPods'; +-- data input s3 bucket parameters +INSERT INTO arc.parameter VALUES ('kubernetes.gui.s3.in.enable','false', 'Enable the use of an s3 input bucket in gui'); +INSERT INTO arc.parameter VALUES ('kubernetes.gui.s3.in.name','s3-name', 'Name of the s3 input bucket used by gui'); +INSERT INTO arc.parameter VALUES ('kubernetes.gui.s3.in.access','s3-access', 'Access identifier for s3 access in gui'); +INSERT INTO arc.parameter VALUES ('kubernetes.gui.s3.in.secret','s3-secret', 'Secret for s3 access in gui'); +INSERT INTO arc.parameter VALUES ('kubernetes.batch.s3.in.enable','false', 'Enable the use of an s3 input bucket in batch'); +INSERT INTO arc.parameter VALUES ('kubernetes.batch.s3.in.name','s3-name', 'Name of the s3 input bucket used by batch'); +INSERT INTO arc.parameter VALUES ('kubernetes.batch.s3.in.access','s3-access', 'Access identifier for s3 access in batch'); +INSERT INTO arc.parameter VALUES ('kubernetes.batch.s3.in.secret','s3-secret', 'Secret for s3 access in batch'); -- patch 23/06/2023 -- remove deprecated parameters @@ -114,6 +118,9 @@ DELETE FROM arc.parameter WHERE key='LanceurARC.MAX_PARALLEL_RUNNER_PER_PHASE'; DELETE FROM arc.parameter WHERE key='LanceurARC.PARALLEL_LOCK_CHECK_INTERVAL'; DELETE FROM arc.parameter WHERE key='LanceurARC.env'; +commit; +vacuum arc.parameter; + -- table de pilotage du batch de production CREATE TABLE IF NOT EXISTS arc.pilotage_batch (last_init text, operation text); insert into arc.pilotage_batch select '1900-01-01:00','O' where not exists (select from arc.pilotage_batch); diff --git a/arc-core/src/test/java/fr/insee/arc/core/businesstest/TestsFonctionnels.java b/arc-core/src/test/java/fr/insee/arc/core/businesstest/TestsFonctionnels.java index af51c5bac..4742bf6ee 100644 --- a/arc-core/src/test/java/fr/insee/arc/core/businesstest/TestsFonctionnels.java +++ b/arc-core/src/test/java/fr/insee/arc/core/businesstest/TestsFonctionnels.java @@ -63,7 +63,7 @@ public void executeFunctionnalTests() throws IOException, SQLException, ArcExcep private void executeTestSirene(String sandbox, String repertoire) throws IOException, ArcException, SQLException { BddPatcherTest.insertTestDataSirene(); - ApiServiceFactory.getService(TraitementPhase.INITIALISATION, sandbox, repertoire, 10000000, null).invokeApi(); + ApiServiceFactory.getService(TraitementPhase.INITIALISATION, sandbox, 10000000, null).invokeApi(); String repertoireDeDepot = DirectoryPath.directoryReceptionEntrepot(repertoire, sandbox, DataWarehouse.DEFAULT.getName()); @@ -73,17 +73,17 @@ private void executeTestSirene(String sandbox, String repertoire) throws IOExcep Files.copy(this.getClass().getClassLoader().getResourceAsStream("testFiles/Cas_test_V2016.02.zip"), new File(repertoireDeDepot, "Cas_test_V2016.02.zip").toPath()); - ApiServiceFactory.getService(TraitementPhase.RECEPTION, sandbox, repertoire, 10000000, null).invokeApi(); + ApiServiceFactory.getService(TraitementPhase.RECEPTION, sandbox, 10000000, null).invokeApi(); assertEquals(114, nbFileInPhase(sandbox, TraitementPhase.RECEPTION, TraitementEtat.OK)); - ApiServiceFactory.getService(TraitementPhase.CHARGEMENT, sandbox, repertoire, 10000000, null).invokeApi(); + ApiServiceFactory.getService(TraitementPhase.CHARGEMENT, sandbox, 10000000, null).invokeApi(); assertEquals(114, nbFileInPhase(sandbox, TraitementPhase.CHARGEMENT, TraitementEtat.OK)); - ApiServiceFactory.getService(TraitementPhase.NORMAGE, sandbox, repertoire, 10000000, null).invokeApi(); + ApiServiceFactory.getService(TraitementPhase.NORMAGE, sandbox, 10000000, null).invokeApi(); assertEquals(114, nbFileInPhase(sandbox, TraitementPhase.NORMAGE, TraitementEtat.OK)); - ApiServiceFactory.getService(TraitementPhase.CONTROLE, sandbox, repertoire, 10000000, null).invokeApi(); + ApiServiceFactory.getService(TraitementPhase.CONTROLE, sandbox, 10000000, null).invokeApi(); assertEquals(47, nbFileInPhase(sandbox, TraitementPhase.CONTROLE, TraitementEtat.OK)); } @@ -104,7 +104,7 @@ private void executeTestSiera(String sandbox, String repertoire) throws IOExcept // to test batch mode String batchMode="1"; - ApiServiceFactory.getService(TraitementPhase.INITIALISATION, sandbox, repertoire, 10000000, batchMode).invokeApi(); + ApiServiceFactory.getService(TraitementPhase.INITIALISATION, sandbox, 10000000, batchMode).invokeApi(); String repertoireDeDepot = DirectoryPath.directoryReceptionEntrepot(repertoire, sandbox, DataWarehouse.DEFAULT.getName()); @@ -112,20 +112,20 @@ private void executeTestSiera(String sandbox, String repertoire) throws IOExcept Files.copy(this.getClass().getClassLoader().getResourceAsStream("testFiles/siera_ano.xml"), new File(repertoireDeDepot, "siera_ano.xml").toPath()); - ApiServiceFactory.getService(TraitementPhase.RECEPTION, sandbox, repertoire, 10000000, batchMode).invokeApi(); + ApiServiceFactory.getService(TraitementPhase.RECEPTION, sandbox, 10000000, batchMode).invokeApi(); assertEquals(1, nbFileInPhase(sandbox, TraitementPhase.RECEPTION, TraitementEtat.OK)); - ApiServiceFactory.getService(TraitementPhase.CHARGEMENT, sandbox, repertoire, 10000000, batchMode).invokeApi(); + ApiServiceFactory.getService(TraitementPhase.CHARGEMENT, sandbox, 10000000, batchMode).invokeApi(); assertEquals(1, nbFileInPhase(sandbox, TraitementPhase.CHARGEMENT, TraitementEtat.OK)); - ApiServiceFactory.getService(TraitementPhase.NORMAGE, sandbox, repertoire, 10000000, batchMode).invokeApi(); + ApiServiceFactory.getService(TraitementPhase.NORMAGE, sandbox, 10000000, batchMode).invokeApi(); assertEquals(1, nbFileInPhase(sandbox, TraitementPhase.NORMAGE, TraitementEtat.OK)); - ApiServiceFactory.getService(TraitementPhase.CONTROLE, sandbox, repertoire, 10000000, batchMode).invokeApi(); + ApiServiceFactory.getService(TraitementPhase.CONTROLE, sandbox, 10000000, batchMode).invokeApi(); assertEquals(1, nbFileInPhase(sandbox, TraitementPhase.CONTROLE, TraitementEtat.OK)); - ApiServiceFactory.getService(TraitementPhase.MAPPING, sandbox, repertoire, 10000000, batchMode).invokeApi(); + ApiServiceFactory.getService(TraitementPhase.MAPPING, sandbox, 10000000, batchMode).invokeApi(); assertEquals(1, nbFileInPhase(sandbox, TraitementPhase.MAPPING, TraitementEtat.OK)); // test on data @@ -167,7 +167,7 @@ private void executeTestSiera(String sandbox, String repertoire) throws IOExcept private void executeTestAnimal(String sandbox, String repertoire) throws IOException, ArcException, SQLException { BddPatcherTest.insertTestDataAnimal(); - ApiServiceFactory.getService(TraitementPhase.INITIALISATION, sandbox, repertoire, 10000000, null).invokeApi(); + ApiServiceFactory.getService(TraitementPhase.INITIALISATION, sandbox, 10000000, null).invokeApi(); String repertoireDeDepot = DirectoryPath.directoryReceptionEntrepot(repertoire, sandbox, DataWarehouse.DEFAULT.getName()); @@ -175,18 +175,18 @@ private void executeTestAnimal(String sandbox, String repertoire) throws IOExcep Files.copy(this.getClass().getClassLoader().getResourceAsStream("testFiles/animals.tar.gz"), new File(repertoireDeDepot, "animals.tar.gz").toPath()); - ApiServiceFactory.getService(TraitementPhase.RECEPTION, sandbox, repertoire, 10000000, null).invokeApi(); + ApiServiceFactory.getService(TraitementPhase.RECEPTION, sandbox, 10000000, null).invokeApi(); assertEquals(2, nbFileInPhase(sandbox, TraitementPhase.RECEPTION, TraitementEtat.OK)); - ApiServiceFactory.getService(TraitementPhase.CHARGEMENT, sandbox, repertoire, 10000000, null).invokeApi(); + ApiServiceFactory.getService(TraitementPhase.CHARGEMENT, sandbox, 10000000, null).invokeApi(); assertEquals(2, nbFileInPhase(sandbox, TraitementPhase.CHARGEMENT, TraitementEtat.OK)); // doublon detection test Files.copy(this.getClass().getClassLoader().getResourceAsStream("testFiles/animals-001.csv"), new File(repertoireDeDepot, "animals-001.csv").toPath()); - ApiServiceFactory.getService(TraitementPhase.RECEPTION, sandbox, repertoire, 10000000, null).invokeApi(); + ApiServiceFactory.getService(TraitementPhase.RECEPTION, sandbox, 10000000, null).invokeApi(); assertEquals(1, nbFileInPhase(sandbox, TraitementPhase.RECEPTION, TraitementEtat.KO)); diff --git a/arc-core/src/test/java/fr/insee/arc/core/factory/ApiServiceFactoryTest.java b/arc-core/src/test/java/fr/insee/arc/core/factory/ApiServiceFactoryTest.java index 40403e576..50a097531 100644 --- a/arc-core/src/test/java/fr/insee/arc/core/factory/ApiServiceFactoryTest.java +++ b/arc-core/src/test/java/fr/insee/arc/core/factory/ApiServiceFactoryTest.java @@ -20,7 +20,7 @@ public void testMegaFactoryInstance() { for (TraitementPhase arcPhase:TraitementPhase.getListPhaseC()) { - p=ApiServiceFactory.getService(arcPhase, "arc_bas2", "./", capacity, null); + p=ApiServiceFactory.getService(arcPhase, "arc_bas2", capacity, null); assertEquals(null, p.getParamBatch()); assertEquals(capacity, p.getNbEnr().intValue()); diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/kubernetes/ApiKubernetesService.java b/arc-utils/src/main/java/fr/insee/arc/utils/kubernetes/KubernetesService.java similarity index 95% rename from arc-core/src/main/java/fr/insee/arc/core/service/kubernetes/ApiKubernetesService.java rename to arc-utils/src/main/java/fr/insee/arc/utils/kubernetes/KubernetesService.java index 4c9cd005a..6d98efe0f 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/kubernetes/ApiKubernetesService.java +++ b/arc-utils/src/main/java/fr/insee/arc/utils/kubernetes/KubernetesService.java @@ -1,4 +1,4 @@ -package fr.insee.arc.core.service.kubernetes; +package fr.insee.arc.utils.kubernetes; import java.io.BufferedReader; import java.io.IOException; @@ -19,9 +19,9 @@ import org.apache.commons.lang3.exception.ExceptionUtils; -import fr.insee.arc.core.service.kubernetes.bo.KubernetesServiceResult; +import fr.insee.arc.utils.kubernetes.bo.KubernetesServiceResult; -public class ApiKubernetesService { +public class KubernetesService { public static KubernetesServiceResult execute(String urlProvided, String httpType, String token, String json) diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/kubernetes/bo/KubernetesServiceResult.java b/arc-utils/src/main/java/fr/insee/arc/utils/kubernetes/bo/KubernetesServiceResult.java similarity index 91% rename from arc-core/src/main/java/fr/insee/arc/core/service/kubernetes/bo/KubernetesServiceResult.java rename to arc-utils/src/main/java/fr/insee/arc/utils/kubernetes/bo/KubernetesServiceResult.java index 6f0945dd6..71abb1052 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/kubernetes/bo/KubernetesServiceResult.java +++ b/arc-utils/src/main/java/fr/insee/arc/utils/kubernetes/bo/KubernetesServiceResult.java @@ -1,4 +1,4 @@ -package fr.insee.arc.core.service.kubernetes.bo; +package fr.insee.arc.utils.kubernetes.bo; public class KubernetesServiceResult { diff --git a/arc-utils/src/main/java/fr/insee/arc/utils/minio/S3Service.java b/arc-utils/src/main/java/fr/insee/arc/utils/minio/S3Service.java new file mode 100644 index 000000000..e56088064 --- /dev/null +++ b/arc-utils/src/main/java/fr/insee/arc/utils/minio/S3Service.java @@ -0,0 +1,9 @@ +package fr.insee.arc.utils.minio; + +public class S3Service { + + + + + +} diff --git a/arc-utils/src/main/java/fr/insee/arc/utils/ressourceUtils/PropertiesHandler.java b/arc-utils/src/main/java/fr/insee/arc/utils/ressourceUtils/PropertiesHandler.java index 53bb93e69..0e66dd6fa 100644 --- a/arc-utils/src/main/java/fr/insee/arc/utils/ressourceUtils/PropertiesHandler.java +++ b/arc-utils/src/main/java/fr/insee/arc/utils/ressourceUtils/PropertiesHandler.java @@ -5,13 +5,13 @@ import java.util.List; import java.util.Map; -import org.springframework.stereotype.Component; +import org.springframework.stereotype.Service; import fr.insee.arc.utils.exception.ArcException; import fr.insee.arc.utils.exception.ArcExceptionMessage; import fr.insee.arc.utils.utils.ManipString; -@Component("properties") +@Service("properties") public class PropertiesHandler { /* Database */ diff --git a/arc-utils/src/main/java/fr/insee/arc/utils/webutils/WebAttributesName.java b/arc-utils/src/main/java/fr/insee/arc/utils/webutils/WebAttributesName.java index 711768621..f97e9f772 100644 --- a/arc-utils/src/main/java/fr/insee/arc/utils/webutils/WebAttributesName.java +++ b/arc-utils/src/main/java/fr/insee/arc/utils/webutils/WebAttributesName.java @@ -3,8 +3,6 @@ import java.util.HashMap; import java.util.Map; -import org.springframework.beans.factory.annotation.Value; - import fr.insee.arc.utils.dao.UtilitaireDao; import fr.insee.arc.utils.ressourceUtils.PropertiesHandler; diff --git a/arc-web/src/main/java/fr/insee/arc/web/gui/all/service/ArcWebGenericService.java b/arc-web/src/main/java/fr/insee/arc/web/gui/all/service/ArcWebGenericService.java index 750abe14c..cf3d1cb5b 100644 --- a/arc-web/src/main/java/fr/insee/arc/web/gui/all/service/ArcWebGenericService.java +++ b/arc-web/src/main/java/fr/insee/arc/web/gui/all/service/ArcWebGenericService.java @@ -49,7 +49,6 @@ public abstract class ArcWebGenericService i protected static final String POOLNAME = "arc"; @Autowired - @Qualifier("properties") protected PropertiesHandler properties; @Autowired @@ -166,10 +165,10 @@ public void initializeModel(@ModelAttribute T arcModel, Model model, loggerDispatcher.info(String.format("env selected %s", bacASable), LOGGER); this.bacASable = bacASable; } - this.isEnvProd = Sandbox.isEnvSetForProduction(this.bacASable); this.dataObjectService.setSandboxSchema(this.bacASable); - this.isKube = Boolean.parseBoolean(new BDParameters(ArcDatabase.COORDINATOR).getString(null, "ArcAction.enableKube","false")); + this.isEnvProd = Sandbox.isEnvSetForProduction(this.bacASable); + this.isKube = Boolean.parseBoolean(new BDParameters(ArcDatabase.COORDINATOR).getString(null, "kubernetes.gui.s3.in.enable","false")); dao.initialize(vObjectService, dataObjectService); diff --git a/arc-web/src/main/java/fr/insee/arc/web/gui/maintenanceoperation/model/ViewKubernetes.java b/arc-web/src/main/java/fr/insee/arc/web/gui/maintenanceoperation/model/ViewKubernetes.java index beb2e5e11..81ad19b6b 100644 --- a/arc-web/src/main/java/fr/insee/arc/web/gui/maintenanceoperation/model/ViewKubernetes.java +++ b/arc-web/src/main/java/fr/insee/arc/web/gui/maintenanceoperation/model/ViewKubernetes.java @@ -14,9 +14,6 @@ public class ViewKubernetes extends VObject { static { columnMap.put("i", new ColumnRendering(false, "i", "0%", "text", null, false)); - columnMap.put("key", new ColumnRendering(true, "label.key", "15%", "text", null, true)); - columnMap.put("val", new ColumnRendering(true, "label.val", "15%", "text", null, true)); - columnMap.put("description", new ColumnRendering(true, "label.description", "70%", "text", null, false)); } public ViewKubernetes() { diff --git a/arc-web/src/main/java/fr/insee/arc/web/gui/maintenanceoperation/model/ViewOperations.java b/arc-web/src/main/java/fr/insee/arc/web/gui/maintenanceoperation/model/ViewOperations.java index bf3c499c2..cd7b676fb 100644 --- a/arc-web/src/main/java/fr/insee/arc/web/gui/maintenanceoperation/model/ViewOperations.java +++ b/arc-web/src/main/java/fr/insee/arc/web/gui/maintenanceoperation/model/ViewOperations.java @@ -7,24 +7,19 @@ import fr.insee.arc.web.gui.all.util.ConstantVObject.ColumnRendering; import fr.insee.arc.web.gui.all.util.VObject; - public class ViewOperations extends VObject { - private static final Map columnMap= new HashMap<>(); + private static final Map columnMap = new HashMap<>(); static { columnMap.put("i", new ColumnRendering(false, "i", "0%", "text", null, false)); - columnMap.put("key", new ColumnRendering(true, "label.key", "15%", "text", null, true)); - columnMap.put("val", new ColumnRendering(true, "label.val", "15%", "text", null, true)); - columnMap.put("description", new ColumnRendering(true, "label.description", "70%", "text", null, false)); - } - + public ViewOperations() { - super(); - this.setTitle("view.operations"); - this.setSessionName("viewOperations"); - this.setDefaultPaginationSize(20); - this.setConstantVObject(new ConstantVObject(columnMap)); - } + super(); + this.setTitle("view.operations"); + this.setSessionName("viewOperations"); + this.setDefaultPaginationSize(20); + this.setConstantVObject(new ConstantVObject(columnMap)); + } } \ No newline at end of file diff --git a/arc-web/src/main/java/fr/insee/arc/web/gui/maintenanceoperation/service/ServiceViewKubernetes.java b/arc-web/src/main/java/fr/insee/arc/web/gui/maintenanceoperation/service/ServiceViewKubernetes.java index 64426d748..b8861e1b0 100644 --- a/arc-web/src/main/java/fr/insee/arc/web/gui/maintenanceoperation/service/ServiceViewKubernetes.java +++ b/arc-web/src/main/java/fr/insee/arc/web/gui/maintenanceoperation/service/ServiceViewKubernetes.java @@ -12,8 +12,8 @@ import org.springframework.stereotype.Service; import org.springframework.ui.Model; -import fr.insee.arc.core.service.kubernetes.ApiKubernetesService; -import fr.insee.arc.core.service.kubernetes.bo.KubernetesServiceResult; +import fr.insee.arc.utils.kubernetes.KubernetesService; +import fr.insee.arc.utils.kubernetes.bo.KubernetesServiceResult; import io.minio.ListObjectsArgs; import io.minio.MinioClient; import io.minio.Result; @@ -31,11 +31,10 @@ public class ServiceViewKubernetes extends InteractorMaintenanceOperations { public String createDatabases(Model model) throws IOException { // récupération du token // kubectl exec -t cat /var/run/secrets/kubernetes.io/serviceaccount/token - String token = "Bearer " - + new String(Files.readAllBytes(Paths.get("/var/run/secrets/kubernetes.io/serviceaccount/token")), + String token = new String(Files.readAllBytes(Paths.get("/var/run/secrets/kubernetes.io/serviceaccount/token")), StandardCharsets.UTF_8); - KubernetesServiceResult result = ApiKubernetesService.execute(views.getUrl(), + KubernetesServiceResult result = KubernetesService.execute(views.getUrl(), views.getHttpType(), token, views.getJson()); views.setHttpOutput(result.getResponse()); @@ -73,7 +72,7 @@ public String testMinio(Model model) throws IOException, KeyManagementException, public String executeService(Model model) { - KubernetesServiceResult result = ApiKubernetesService.execute(views.getUrl(), + KubernetesServiceResult result = KubernetesService.execute(views.getUrl(), views.getHttpType(), views.getToken(), views.getJson()); views.setHttpOutput(result.getResponse()); model.addAttribute("httpOutput", views.getHttpOutput()); diff --git a/arc-web/src/main/java/fr/insee/arc/web/gui/pilotage/service/ServiceViewFichierBAS.java b/arc-web/src/main/java/fr/insee/arc/web/gui/pilotage/service/ServiceViewFichierBAS.java index 08f8d668f..88b993833 100644 --- a/arc-web/src/main/java/fr/insee/arc/web/gui/pilotage/service/ServiceViewFichierBAS.java +++ b/arc-web/src/main/java/fr/insee/arc/web/gui/pilotage/service/ServiceViewFichierBAS.java @@ -81,12 +81,10 @@ private String restore(Model model, String code, String messageOk) { // Lancement de l'initialisation dans la foulée loggerDispatcher.info("Synchronisation de l'environnement ", LOGGER); ApiServiceFactory - .getService(TraitementPhase.INITIALISATION, getBacASable(), - this.repertoire, TraitementPhase.INITIALISATION.getNbLigneATraiter(), null) + .getService(TraitementPhase.INITIALISATION, getBacASable(), TraitementPhase.INITIALISATION.getNbLigneATraiter(), null) .invokeApi(); ApiServiceFactory - .getService(TraitementPhase.RECEPTION, getBacASable(), - this.repertoire, TraitementPhase.RECEPTION.getNbLigneATraiter(), null) + .getService(TraitementPhase.RECEPTION, getBacASable(), TraitementPhase.RECEPTION.getNbLigneATraiter(), null) .invokeApi(); } this.views.getViewPilotageBAS().setMessage(message); @@ -173,8 +171,7 @@ public String toDeleteBAS(Model model) { if (!isEnvProd()) { loggerDispatcher.info("Synchronisation de l'environnement ", LOGGER); ApiServiceFactory - .getService(TraitementPhase.INITIALISATION, getBacASable(), - this.repertoire, TraitementPhase.INITIALISATION.getNbLigneATraiter(), null) + .getService(TraitementPhase.INITIALISATION, getBacASable(), TraitementPhase.INITIALISATION.getNbLigneATraiter(), null) .invokeApi(); } diff --git a/arc-web/src/main/java/fr/insee/arc/web/gui/pilotage/service/ServiceViewPilotageBAS.java b/arc-web/src/main/java/fr/insee/arc/web/gui/pilotage/service/ServiceViewPilotageBAS.java index b544dad33..d7fb2aa6c 100644 --- a/arc-web/src/main/java/fr/insee/arc/web/gui/pilotage/service/ServiceViewPilotageBAS.java +++ b/arc-web/src/main/java/fr/insee/arc/web/gui/pilotage/service/ServiceViewPilotageBAS.java @@ -82,11 +82,11 @@ public String filesUploadBAS(Model model) { // Lancement de l'initialisation dans la foulée ApiServiceFactory .getService(TraitementPhase.INITIALISATION, getBacASable(), - this.repertoire, TraitementPhase.INITIALISATION.getNbLigneATraiter(), null) + TraitementPhase.INITIALISATION.getNbLigneATraiter(), null) .invokeApi(); ApiServiceFactory .getService(TraitementPhase.RECEPTION, getBacASable(), - this.repertoire, TraitementPhase.RECEPTION.getNbLigneATraiter(), null) + TraitementPhase.RECEPTION.getNbLigneATraiter(), null) .invokeApi(); } @@ -182,7 +182,7 @@ public String executerBatch(Model model, TraitementPhase phaseAExecuter) { // Maximum number of files processed in each phase iteration int maxFilesPerPhase = new BDParameters(ArcDatabase.COORDINATOR).getInt(null, "LanceurIHM.maxFilesPerPhase", 10000000); - ApiServiceFactory.getService(phaseAExecuter, getBacASable(), this.repertoire, + ApiServiceFactory.getService(phaseAExecuter, getBacASable(), maxFilesPerPhase, null ).invokeApi(); return generateDisplay(model, RESULT_SUCCESS); diff --git a/arc-ws/src/main/java/fr/insee/arc/ws/services/restServices/execute/operation/ExecuteServiceOperation.java b/arc-ws/src/main/java/fr/insee/arc/ws/services/restServices/execute/operation/ExecuteServiceOperation.java index be9b7af7b..9f4e8ebfe 100644 --- a/arc-ws/src/main/java/fr/insee/arc/ws/services/restServices/execute/operation/ExecuteServiceOperation.java +++ b/arc-ws/src/main/java/fr/insee/arc/ws/services/restServices/execute/operation/ExecuteServiceOperation.java @@ -65,7 +65,7 @@ public void executeServiceClient() throws ArcException { } - ApiServiceFactory.getService(TraitementPhase.getPhase(bodyPojo.targetPhase), env, repertoire, + ApiServiceFactory.getService(TraitementPhase.getPhase(bodyPojo.targetPhase), env, Integer.MAX_VALUE, null).invokeApi(); ExecuteRulesDao.buildResponse(connection, bodyPojo, returnView, responseAttributes.getFirstContactDate());