Skip to content

Commit

Permalink
feat: removed deprecated directory parameter in api
Browse files Browse the repository at this point in the history
  • Loading branch information
Nolife999 committed Feb 6, 2024
1 parent 925b8fc commit 7e41e5d
Show file tree
Hide file tree
Showing 32 changed files with 99 additions and 101 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ public void execute() {
this.report = ApiServiceFactory.getService( //
getPhaseName(), //
mapParam.get(PhaseParameterKeys.KEY_FOR_EXECUTION_ENVIRONMENT), //
mapParam.get(PhaseParameterKeys.KEY_FOR_DIRECTORY_LOCATION), //
Integer.parseInt(mapParam.get(capacityParameterName())), //
Boolean.parseBoolean(PhaseParameterKeys.KEY_FOR_KEEP_IN_DATABASE) ? null
: mapParam.get(PhaseParameterKeys.KEY_FOR_BATCH_CHUNK_ID)) //
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ public class ApiChargementServiceFactory implements IServiceFactory {
* @param aDirectoryRoot
* @param aNbEnr
*/
public ApiService get(TraitementPhase phaseService, String executionSchema, String directory, Integer capacityParameter, String paramBatch) {
return new ApiChargementService(phaseService, executionSchema, directory, capacityParameter, paramBatch);
public ApiService get(TraitementPhase phaseService, String executionSchema, Integer capacityParameter, String paramBatch) {
return new ApiChargementService(phaseService, executionSchema, capacityParameter, paramBatch);
}

public static IServiceFactory getInstance() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ public class ApiControleServiceFactory implements IServiceFactory {
* @param aDirectoryRoot
* @param aNbEnr
*/
public ApiService get(TraitementPhase phaseService, String executionSchema, String directory, Integer capacityParameter, String paramBatch) {
return new ApiControleService(phaseService, executionSchema, directory, capacityParameter, paramBatch);
public ApiService get(TraitementPhase phaseService, String executionSchema, Integer capacityParameter, String paramBatch) {
return new ApiControleService(phaseService, executionSchema, capacityParameter, paramBatch);
}

public static IServiceFactory getInstance() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
public class ApiInitialisationServiceFactory implements IServiceFactory {

@Override
public ApiService get(TraitementPhase phaseService, String executionSchema, String directory, Integer capacityParameter, String paramBatch) {
return new ApiInitialisationService(phaseService, executionSchema, directory, capacityParameter, paramBatch);
public ApiService get(TraitementPhase phaseService, String executionSchema, Integer capacityParameter, String paramBatch) {
return new ApiInitialisationService(phaseService, executionSchema, capacityParameter, paramBatch);
}

public static IServiceFactory getInstance() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
public class ApiMappingServiceFactory implements IServiceFactory {

@Override
public ApiService get(TraitementPhase phaseService, String executionSchema, String directory, Integer capacityParameter, String paramBatch) {
return new ApiMappingService(phaseService, executionSchema, directory, capacityParameter, paramBatch);
public ApiService get(TraitementPhase phaseService, String executionSchema, Integer capacityParameter, String paramBatch) {
return new ApiMappingService(phaseService, executionSchema, capacityParameter, paramBatch);
}

public static IServiceFactory getInstance() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ public class ApiNormageServiceFactory implements IServiceFactory {
* @param aDirectoryRoot
* @param aNbEnr
*/
public ApiService get(TraitementPhase phaseService, String executionSchema, String directory, Integer capacityParameter, String paramBatch) {
return new ApiNormageService(phaseService, executionSchema, directory, capacityParameter, paramBatch);
public ApiService get(TraitementPhase phaseService, String executionSchema, Integer capacityParameter, String paramBatch) {
return new ApiNormageService(phaseService, executionSchema, capacityParameter, paramBatch);
}

public static IServiceFactory getInstance() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ public class ApiReceptionServiceFactory implements IServiceFactory {
* @param aDirectoryRoot
* @param aNbEnr
*/
public ApiService get(TraitementPhase phaseService, String executionSchema, String directory, Integer capacityParameter, String paramBatch) {
return new ApiReceptionService(phaseService, executionSchema, directory, capacityParameter, paramBatch);
public ApiService get(TraitementPhase phaseService, String executionSchema, Integer capacityParameter, String paramBatch) {
return new ApiReceptionService(phaseService, executionSchema, capacityParameter, paramBatch);
}

public static IServiceFactory getInstance() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ private static final ApiServiceFactory getInstance() {
* @param paramBatch
* @return
*/
public static final ApiService getService(TraitementPhase phaseService, String executionSchema, String directory, Integer capacityParameter, String paramBatch) {
return getInstance().map.get(phaseService).get(phaseService, executionSchema, directory, capacityParameter, paramBatch);
public static final ApiService getService(TraitementPhase phaseService, String executionSchema, Integer capacityParameter, String paramBatch) {
return getInstance().map.get(phaseService).get(phaseService, executionSchema, capacityParameter, paramBatch);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,6 @@ public interface IServiceFactory {
* @param paramBatch
* @return
*/
public ApiService get(TraitementPhase phaseService, String executionSchema, String directory, Integer capacityParameter, String paramBatch);
public ApiService get(TraitementPhase phaseService, String executionSchema, Integer capacityParameter, String paramBatch);

}
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ public ApiService() {
springInit();
}

protected ApiService(TraitementPhase aCurrentPhase, String aEnvExecution, String aDirectoryRoot, Integer aNbEnr,
protected ApiService(TraitementPhase aCurrentPhase, String aEnvExecution, Integer aNbEnr,
String paramBatch) {

StaticLoggerDispatcher.info(LOGGER_APISERVICE, "** initialiserVariable **");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@ public ApiInitialisationService() {
super();
}

public ApiInitialisationService(TraitementPhase aCurrentPhase, String aEnvExecution, String aDirectoryRoot, Integer aNbEnr,
public ApiInitialisationService(TraitementPhase aCurrentPhase, String aEnvExecution, Integer aNbEnr,
String paramBatch) {
super(aCurrentPhase, aEnvExecution, aDirectoryRoot, aNbEnr, paramBatch);
super(aCurrentPhase, aEnvExecution, aNbEnr, paramBatch);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,7 @@ public static void backToTargetPhase(TraitementPhase phaseAExecuter, String env,
if (phaseAExecuter.getOrdre() == TraitementPhase.INITIALISATION.getOrdre()) {
resetBAS(env, rootDirectory);
} else {
ApiInitialisationService serv = new ApiInitialisationService(TraitementPhase.INITIALISATION, env,
rootDirectory, TraitementPhase.INITIALISATION.getNbLigneATraiter(), null);
ApiInitialisationService serv = new ApiInitialisationService(TraitementPhase.INITIALISATION, env, TraitementPhase.INITIALISATION.getNbLigneATraiter(), null);
try {
new ResetEnvironmentOperation(serv.getCoordinatorSandbox()).retourPhasePrecedente(phaseAExecuter, undoFilesSelection);
} finally {
Expand All @@ -51,8 +50,7 @@ public static void backToTargetPhase(TraitementPhase phaseAExecuter, String env,
*/
public static void resetBAS(String env, String rootDirectory) {

ApiInitialisationService service = new ApiInitialisationService(TraitementPhase.INITIALISATION, env,
rootDirectory, TraitementPhase.INITIALISATION.getNbLigneATraiter(), null);
ApiInitialisationService service = new ApiInitialisationService(TraitementPhase.INITIALISATION, env, TraitementPhase.INITIALISATION.getNbLigneATraiter(), null);
try {
// delete files and pilotage tables
new ResetEnvironmentOperation(service.getCoordinatorSandbox()).clearPilotageAndDirectories(rootDirectory);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ public ApiReceptionService() {
}

public ApiReceptionService(TraitementPhase aCurrentPhase, String aEnvExecution,
String aDirectoryRoot, Integer aNbEnr, String paramBatch) {
super(aCurrentPhase, aEnvExecution, aDirectoryRoot, aNbEnr, paramBatch);
Integer aNbEnr, String paramBatch) {
super(aCurrentPhase, aEnvExecution, aNbEnr, paramBatch);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,9 @@ public ApiChargementService() {
super();
}

public ApiChargementService(TraitementPhase aCurrentPhase, String aEnvExecution, String aDirectoryRoot, Integer aNbEnr,
public ApiChargementService(TraitementPhase aCurrentPhase, String aEnvExecution, Integer aNbEnr,
String paramBatch) {
super(aCurrentPhase, aEnvExecution, aDirectoryRoot, aNbEnr, paramBatch);
super(aCurrentPhase, aEnvExecution, aNbEnr, paramBatch);
}

protected List<NormeRules> listeNorme;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,9 @@ public ApiNormageService() {
super();
}

public ApiNormageService(TraitementPhase aCurrentPhase, String aEnvExecution, String aDirectoryRoot, Integer aNbEnr,
public ApiNormageService(TraitementPhase aCurrentPhase, String aEnvExecution, Integer aNbEnr,
String paramBatch) {
super(aCurrentPhase, aEnvExecution, aDirectoryRoot, aNbEnr, paramBatch);
super(aCurrentPhase, aEnvExecution, aNbEnr, paramBatch);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,9 @@ public ApiControleService() {
super();
}

public ApiControleService(TraitementPhase aCurrentPhase, String aEnvExecution, String aDirectoryRoot, Integer aNbEnr,
public ApiControleService(TraitementPhase aCurrentPhase, String aEnvExecution, Integer aNbEnr,
String paramBatch) {
super(aCurrentPhase, aEnvExecution, aDirectoryRoot, aNbEnr, paramBatch);
super(aCurrentPhase, aEnvExecution, aNbEnr, paramBatch);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,9 @@ public ApiMappingService() {
* @param aCurrentPhase
* @param aNbEnr
*/
public ApiMappingService(TraitementPhase aCurrentPhase, String anEnvironnementExecution, String aDirectoryRoot,
public ApiMappingService(TraitementPhase aCurrentPhase, String anEnvironnementExecution,
Integer aNbEnr, String paramBatch) {
super(aCurrentPhase, anEnvironnementExecution, null, aNbEnr, paramBatch);
super(aCurrentPhase, anEnvironnementExecution, aNbEnr, paramBatch);
}

/**
Expand Down
25 changes: 16 additions & 9 deletions arc-core/src/main/resources/BdD/script_global.sql
Original file line number Diff line number Diff line change
Expand Up @@ -95,25 +95,32 @@ INSERT INTO arc.parameter VALUES ('MappingService.MAX_PARALLEL_WORKERS','4');
UPDATE arc.parameter set description='parameter.parallel.numberOfThread.p4.mapmodel' where key='MappingService.MAX_PARALLEL_WORKERS';

-- kubernetes parameters
INSERT INTO arc.parameter VALUES ('ArcAction.enableKube','false');
UPDATE arc.parameter set description='parameter.ihm.enableKube' where key='ArcAction.enableKube';

INSERT INTO arc.parameter VALUES ('ArcAction.nbExecutorPods','10');
UPDATE arc.parameter set description='parameter.ihm.nbExecutorPods' where key='ArcAction.nbExecutorPods';
-- horizontal scalability parameters
INSERT INTO arc.parameter VALUES ('kubernetes.gui.scalability.enable','false', 'Enable kubernetes horizontal scalability in gui mode');
INSERT INTO arc.parameter VALUES ('kubernetes.batch.scalability.enable','false', 'Enable kubernetes horizontal scalability in batch mode');
INSERT INTO arc.parameter VALUES ('kubernetes.batch.scalability.numberOfExecutorDatabase','4', 'Number of executor database nod for horizontal scaling');

INSERT INTO arc.parameter VALUES ('BatchArc.enableKube','false');
UPDATE arc.parameter set description='parameter.batch.enableKube' where key='BatchArc.enableKube';

INSERT INTO arc.parameter VALUES ('BatchArc.nbExecutorPods','10');
UPDATE arc.parameter set description='parameter.batch.nbExecutorPods' where key='BatchArc.nbExecutorPods';
-- data input s3 bucket parameters
INSERT INTO arc.parameter VALUES ('kubernetes.gui.s3.in.enable','false', 'Enable the use of an s3 input bucket in gui');
INSERT INTO arc.parameter VALUES ('kubernetes.gui.s3.in.name','s3-name', 'Name of the s3 input bucket used by gui');
INSERT INTO arc.parameter VALUES ('kubernetes.gui.s3.in.access','s3-access', 'Access identifier for s3 access in gui');
INSERT INTO arc.parameter VALUES ('kubernetes.gui.s3.in.secret','s3-secret', 'Secret for s3 access in gui');

INSERT INTO arc.parameter VALUES ('kubernetes.batch.s3.in.enable','false', 'Enable the use of an s3 input bucket in batch');
INSERT INTO arc.parameter VALUES ('kubernetes.batch.s3.in.name','s3-name', 'Name of the s3 input bucket used by batch');
INSERT INTO arc.parameter VALUES ('kubernetes.batch.s3.in.access','s3-access', 'Access identifier for s3 access in batch');
INSERT INTO arc.parameter VALUES ('kubernetes.batch.s3.in.secret','s3-secret', 'Secret for s3 access in batch');

-- patch 23/06/2023
-- remove deprecated parameters
DELETE FROM arc.parameter WHERE key='LanceurARC.MAX_PARALLEL_RUNNER_PER_PHASE';
DELETE FROM arc.parameter WHERE key='LanceurARC.PARALLEL_LOCK_CHECK_INTERVAL';
DELETE FROM arc.parameter WHERE key='LanceurARC.env';

commit;
vacuum arc.parameter;

-- table de pilotage du batch de production
CREATE TABLE IF NOT EXISTS arc.pilotage_batch (last_init text, operation text);
insert into arc.pilotage_batch select '1900-01-01:00','O' where not exists (select from arc.pilotage_batch);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ public void executeFunctionnalTests() throws IOException, SQLException, ArcExcep
private void executeTestSirene(String sandbox, String repertoire) throws IOException, ArcException, SQLException {
BddPatcherTest.insertTestDataSirene();

ApiServiceFactory.getService(TraitementPhase.INITIALISATION, sandbox, repertoire, 10000000, null).invokeApi();
ApiServiceFactory.getService(TraitementPhase.INITIALISATION, sandbox, 10000000, null).invokeApi();

String repertoireDeDepot = DirectoryPath.directoryReceptionEntrepot(repertoire, sandbox,
DataWarehouse.DEFAULT.getName());
Expand All @@ -73,17 +73,17 @@ private void executeTestSirene(String sandbox, String repertoire) throws IOExcep
Files.copy(this.getClass().getClassLoader().getResourceAsStream("testFiles/Cas_test_V2016.02.zip"),
new File(repertoireDeDepot, "Cas_test_V2016.02.zip").toPath());

ApiServiceFactory.getService(TraitementPhase.RECEPTION, sandbox, repertoire, 10000000, null).invokeApi();
ApiServiceFactory.getService(TraitementPhase.RECEPTION, sandbox, 10000000, null).invokeApi();

assertEquals(114, nbFileInPhase(sandbox, TraitementPhase.RECEPTION, TraitementEtat.OK));

ApiServiceFactory.getService(TraitementPhase.CHARGEMENT, sandbox, repertoire, 10000000, null).invokeApi();
ApiServiceFactory.getService(TraitementPhase.CHARGEMENT, sandbox, 10000000, null).invokeApi();
assertEquals(114, nbFileInPhase(sandbox, TraitementPhase.CHARGEMENT, TraitementEtat.OK));

ApiServiceFactory.getService(TraitementPhase.NORMAGE, sandbox, repertoire, 10000000, null).invokeApi();
ApiServiceFactory.getService(TraitementPhase.NORMAGE, sandbox, 10000000, null).invokeApi();
assertEquals(114, nbFileInPhase(sandbox, TraitementPhase.NORMAGE, TraitementEtat.OK));

ApiServiceFactory.getService(TraitementPhase.CONTROLE, sandbox, repertoire, 10000000, null).invokeApi();
ApiServiceFactory.getService(TraitementPhase.CONTROLE, sandbox, 10000000, null).invokeApi();
assertEquals(47, nbFileInPhase(sandbox, TraitementPhase.CONTROLE, TraitementEtat.OK));

}
Expand All @@ -104,28 +104,28 @@ private void executeTestSiera(String sandbox, String repertoire) throws IOExcept
// to test batch mode
String batchMode="1";

ApiServiceFactory.getService(TraitementPhase.INITIALISATION, sandbox, repertoire, 10000000, batchMode).invokeApi();
ApiServiceFactory.getService(TraitementPhase.INITIALISATION, sandbox, 10000000, batchMode).invokeApi();

String repertoireDeDepot = DirectoryPath.directoryReceptionEntrepot(repertoire, sandbox,
DataWarehouse.DEFAULT.getName());

Files.copy(this.getClass().getClassLoader().getResourceAsStream("testFiles/siera_ano.xml"),
new File(repertoireDeDepot, "siera_ano.xml").toPath());

ApiServiceFactory.getService(TraitementPhase.RECEPTION, sandbox, repertoire, 10000000, batchMode).invokeApi();
ApiServiceFactory.getService(TraitementPhase.RECEPTION, sandbox, 10000000, batchMode).invokeApi();

assertEquals(1, nbFileInPhase(sandbox, TraitementPhase.RECEPTION, TraitementEtat.OK));

ApiServiceFactory.getService(TraitementPhase.CHARGEMENT, sandbox, repertoire, 10000000, batchMode).invokeApi();
ApiServiceFactory.getService(TraitementPhase.CHARGEMENT, sandbox, 10000000, batchMode).invokeApi();
assertEquals(1, nbFileInPhase(sandbox, TraitementPhase.CHARGEMENT, TraitementEtat.OK));

ApiServiceFactory.getService(TraitementPhase.NORMAGE, sandbox, repertoire, 10000000, batchMode).invokeApi();
ApiServiceFactory.getService(TraitementPhase.NORMAGE, sandbox, 10000000, batchMode).invokeApi();
assertEquals(1, nbFileInPhase(sandbox, TraitementPhase.NORMAGE, TraitementEtat.OK));

ApiServiceFactory.getService(TraitementPhase.CONTROLE, sandbox, repertoire, 10000000, batchMode).invokeApi();
ApiServiceFactory.getService(TraitementPhase.CONTROLE, sandbox, 10000000, batchMode).invokeApi();
assertEquals(1, nbFileInPhase(sandbox, TraitementPhase.CONTROLE, TraitementEtat.OK));

ApiServiceFactory.getService(TraitementPhase.MAPPING, sandbox, repertoire, 10000000, batchMode).invokeApi();
ApiServiceFactory.getService(TraitementPhase.MAPPING, sandbox, 10000000, batchMode).invokeApi();
assertEquals(1, nbFileInPhase(sandbox, TraitementPhase.MAPPING, TraitementEtat.OK));

// test on data
Expand Down Expand Up @@ -167,26 +167,26 @@ private void executeTestSiera(String sandbox, String repertoire) throws IOExcept
private void executeTestAnimal(String sandbox, String repertoire) throws IOException, ArcException, SQLException {
BddPatcherTest.insertTestDataAnimal();

ApiServiceFactory.getService(TraitementPhase.INITIALISATION, sandbox, repertoire, 10000000, null).invokeApi();
ApiServiceFactory.getService(TraitementPhase.INITIALISATION, sandbox, 10000000, null).invokeApi();

String repertoireDeDepot = DirectoryPath.directoryReceptionEntrepot(repertoire, sandbox,
DataWarehouse.DEFAULT.getName());

Files.copy(this.getClass().getClassLoader().getResourceAsStream("testFiles/animals.tar.gz"),
new File(repertoireDeDepot, "animals.tar.gz").toPath());

ApiServiceFactory.getService(TraitementPhase.RECEPTION, sandbox, repertoire, 10000000, null).invokeApi();
ApiServiceFactory.getService(TraitementPhase.RECEPTION, sandbox, 10000000, null).invokeApi();

assertEquals(2, nbFileInPhase(sandbox, TraitementPhase.RECEPTION, TraitementEtat.OK));

ApiServiceFactory.getService(TraitementPhase.CHARGEMENT, sandbox, repertoire, 10000000, null).invokeApi();
ApiServiceFactory.getService(TraitementPhase.CHARGEMENT, sandbox, 10000000, null).invokeApi();
assertEquals(2, nbFileInPhase(sandbox, TraitementPhase.CHARGEMENT, TraitementEtat.OK));

// doublon detection test
Files.copy(this.getClass().getClassLoader().getResourceAsStream("testFiles/animals-001.csv"),
new File(repertoireDeDepot, "animals-001.csv").toPath());

ApiServiceFactory.getService(TraitementPhase.RECEPTION, sandbox, repertoire, 10000000, null).invokeApi();
ApiServiceFactory.getService(TraitementPhase.RECEPTION, sandbox, 10000000, null).invokeApi();

assertEquals(1, nbFileInPhase(sandbox, TraitementPhase.RECEPTION, TraitementEtat.KO));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ public void testMegaFactoryInstance() {

for (TraitementPhase arcPhase:TraitementPhase.getListPhaseC())
{
p=ApiServiceFactory.getService(arcPhase, "arc_bas2", "./", capacity, null);
p=ApiServiceFactory.getService(arcPhase, "arc_bas2", capacity, null);

assertEquals(null, p.getParamBatch());
assertEquals(capacity, p.getNbEnr().intValue());
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package fr.insee.arc.core.service.kubernetes;
package fr.insee.arc.utils.kubernetes;

import java.io.BufferedReader;
import java.io.IOException;
Expand All @@ -19,9 +19,9 @@

import org.apache.commons.lang3.exception.ExceptionUtils;

import fr.insee.arc.core.service.kubernetes.bo.KubernetesServiceResult;
import fr.insee.arc.utils.kubernetes.bo.KubernetesServiceResult;

public class ApiKubernetesService {
public class KubernetesService {


public static KubernetesServiceResult execute(String urlProvided, String httpType, String token, String json)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package fr.insee.arc.core.service.kubernetes.bo;
package fr.insee.arc.utils.kubernetes.bo;

public class KubernetesServiceResult {

Expand Down
Loading

0 comments on commit 7e41e5d

Please sign in to comment.