Skip to content

Commit

Permalink
initialization refactor
Browse files Browse the repository at this point in the history
  • Loading branch information
Nolife999 committed Sep 21, 2023
1 parent 5d803ee commit d0453cc
Show file tree
Hide file tree
Showing 49 changed files with 1,863 additions and 1,700 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import java.util.Map;

import fr.insee.arc.utils.dao.GenericPreparedStatementBuilder;
import fr.insee.arc.utils.dao.SQL;

public class ArcPreparedStatementBuilder extends GenericPreparedStatementBuilder {

Expand Down Expand Up @@ -63,11 +64,24 @@ public StringBuilder sqlListeOfColumnsFromModel(ViewEnum tableEnum) {
return sqlListeOfColumnsFromModel(tableEnum.getColumns());
}

public ArcPreparedStatementBuilder append(ViewEnum view) {
return (ArcPreparedStatementBuilder) this.append(view.getTableName());
}

public ArcPreparedStatementBuilder append(ColumnEnum column) {
return (ArcPreparedStatementBuilder) this.append(column.getColumnName());
}

public String quoteText(ColumnEnum column) {
return quoteText(column.getColumnName());
}

@Override
public ArcPreparedStatementBuilder build(Object... queryElements) {
for (Object queryElement : queryElements) {
getQuery().append(queryElement);
}
return this;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import java.util.List;
import java.util.stream.Collectors;

import fr.insee.arc.utils.dao.SQL;
import fr.insee.arc.utils.dataobjects.PgColumnEnum;
import fr.insee.arc.utils.dataobjects.PgSchemaEnum;
import fr.insee.arc.utils.dataobjects.TypeEnum;
Expand Down Expand Up @@ -123,6 +124,8 @@ public enum ColumnEnum {
, JOINTURE("jointure", TypeEnum.TEXT, "data structure query of the processing file")
, GENERATION_COMPOSITE("generation_composite", TypeEnum.TEXT, "timestamp of the end of the process of the file in pilotage")

, ENTREPOT("entrepot", TypeEnum.TEXT, "name of the datastore that recieve archive file")
, NOM_ARCHIVE("nom_archive", TypeEnum.TEXT, "name of archive file")

, TEST1(PgColumnEnum.TEST1), TEST2(PgColumnEnum.TEST2)

Expand Down Expand Up @@ -187,5 +190,9 @@ public static SchemaEnum convert(PgSchemaEnum pgSchema)
return SchemaEnum.valueOf(pgSchema.toString());
}

public String alias(ViewEnum v)
{
return v.getTableName()+SQL.DOT.getSqlCode()+this.columnName;
}

}
Original file line number Diff line number Diff line change
@@ -1,12 +1,8 @@
package fr.insee.arc.core.dataobjects;

public class DataObjectService {
import fr.insee.arc.utils.dao.SQL;

/**
* database characters convention
*/
public static final String SCHEMA_SEPARATOR = ".";
public static final String QUERY_SEPARATOR = ";";
public class DataObjectService {

/**
* ARC special parameter
Expand Down Expand Up @@ -40,15 +36,15 @@ public String getView(ViewEnum e) {
}
if ((e.getTableLocation().equals(SchemaEnum.SANDBOX)
|| e.getTableLocation().equals(SchemaEnum.SANDBOX_GENERATED)) && this.sandboxSchema != null) {
return this.sandboxSchema + SCHEMA_SEPARATOR + e.getTableName();
return this.sandboxSchema + SQL.DOT.getSqlCode() + e.getTableName();
}

return getFullTableNameInSchema(e.getTableLocation(), e.getTableName());

}

public static String getFullTableNameInSchema(SchemaEnum schema, String tablename) {
return schema.getSchemaName().equals("") ? tablename : schema.getSchemaName() + SCHEMA_SEPARATOR + tablename;
return schema.getSchemaName().equals("") ? tablename : schema.getSchemaName() + SQL.DOT.getSqlCode() + tablename;
}

public String getSandboxSchema() {
Expand Down
29 changes: 26 additions & 3 deletions arc-core/src/main/java/fr/insee/arc/core/dataobjects/ViewEnum.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import java.util.LinkedHashMap;
import java.util.Map;

import fr.insee.arc.utils.dao.SQL;
import fr.insee.arc.utils.dataobjects.PgColumnEnum;
import fr.insee.arc.utils.dataobjects.PgViewEnum;

Expand Down Expand Up @@ -104,8 +105,8 @@ public enum ViewEnum {
ColumnEnum.DATE_ENTREE, ColumnEnum.CONTAINER, ColumnEnum.V_CONTAINER, ColumnEnum.O_CONTAINER,
ColumnEnum.TO_DELETE, ColumnEnum.CLIENT, ColumnEnum.DATE_CLIENT, ColumnEnum.JOINTURE,
ColumnEnum.GENERATION_COMPOSITE) //
, PILOTAGE_ARCHIVE("pilotage_archive", SchemaEnum.SANDBOX)

, PILOTAGE_ARCHIVE("pilotage_archive", SchemaEnum.SANDBOX, ColumnEnum.ENTREPOT , ColumnEnum.NOM_ARCHIVE)

// family model table in sandbox
, MOD_TABLE_METIER("mod_table_metier", SchemaEnum.SANDBOX_GENERATED, ColumnEnum.ID_FAMILLE,
Expand Down Expand Up @@ -133,7 +134,11 @@ public enum ViewEnum {
TABLE_TEST_OUT_TEMPORARY(PgViewEnum.TABLE_TEST_OUT_TEMPORARY)

// view for table aliases or temporary table in query
, T1(PgViewEnum.T1), T2(PgViewEnum.T1), T3(PgViewEnum.T1)
, T1(PgViewEnum.T1), T2(PgViewEnum.T2), T3(PgViewEnum.T3)

, ALIAS_A(PgViewEnum.ALIAS_A), ALIAS_B(PgViewEnum.ALIAS_B), ALIAS_C(PgViewEnum.ALIAS_C)

, TMP_FILES("tmp_files", SchemaEnum.TEMPORARY, ColumnEnum.FILE_NAME)

, PG_TABLES(PgViewEnum.PG_TABLES)

Expand Down Expand Up @@ -183,4 +188,22 @@ public Map<ColumnEnum, ColumnEnum> getColumns() {
return columns;
}

public String getFullName() {
return DataObjectService.getFullTableNameInSchema(this.tableLocation, this.tableName);
}

public String getFullName(String schema) {
return schema + SQL.DOT.getSqlCode() + this.tableName;
}

public ColumnEnum col(ColumnEnum e) {
return this.getColumns().get(e);
}

@Override
public String toString()
{
return this.getTableName();
}

}
Original file line number Diff line number Diff line change
@@ -1,21 +1,24 @@
package fr.insee.arc.core.model;

import fr.insee.arc.core.dataobjects.ViewEnum;

public enum TraitementTableParametre {
CALENDRIER("CALENDRIER"), //
NORME("NORME"), //
JEUDEREGLE("JEUDEREGLE"), //
CHARGEMENT_REGLE("CHARGEMENT_REGLE"), //
NORMAGE_REGLE("NORMAGE_REGLE"), //
CONTROLE_REGLE("CONTROLE_REGLE"), //
MAPPING_REGLE("MAPPING_REGLE"), //
EXPRESSION("EXPRESSION"), //
MOD_TABLE_METIER("MOD_TABLE_METIER"), //
MOD_VARIABLE_METIER("MOD_VARIABLE_METIER");
private TraitementTableParametre(String anExpression) {
this.expression = anExpression;
CALENDRIER(ViewEnum.IHM_CALENDRIER), //
NORME(ViewEnum.IHM_NORME), //
JEUDEREGLE(ViewEnum.IHM_JEUDEREGLE), //
CHARGEMENT_REGLE(ViewEnum.IHM_CHARGEMENT_REGLE), //
NORMAGE_REGLE(ViewEnum.IHM_NORMAGE_REGLE), //
CONTROLE_REGLE(ViewEnum.IHM_CONTROLE_REGLE), //
MAPPING_REGLE(ViewEnum.IHM_MAPPING_REGLE), //
EXPRESSION(ViewEnum.IHM_EXPRESSION), //
MOD_TABLE_METIER(ViewEnum.MOD_TABLE_METIER), //
MOD_VARIABLE_METIER(ViewEnum.MOD_VARIABLE_METIER);

private TraitementTableParametre(ViewEnum tablename) {
this.tablename = tablename;
}

private String expression;
private ViewEnum tablename;

public boolean isPartOfRuleset() {
return this == TraitementTableParametre.CHARGEMENT_REGLE
Expand All @@ -27,6 +30,6 @@ public boolean isPartOfRuleset() {

@Override
public String toString() {
return this.expression;
return this.tablename.getFullName();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import fr.insee.arc.core.model.TraitementPhase;
import fr.insee.arc.core.model.TraitementTableExecution;
import fr.insee.arc.core.model.TraitementTableParametre;
import fr.insee.arc.core.service.global.bo.Sandbox;
import fr.insee.arc.core.service.global.dao.DatabaseConnexionConfiguration;
import fr.insee.arc.core.service.global.dao.PilotageOperations;
import fr.insee.arc.core.service.global.dao.TableNaming;
Expand Down Expand Up @@ -69,6 +70,8 @@ public abstract class ApiService implements IConstanteNumerique {
protected String currentIdSource;
protected String directoryIn;
protected List<Norme> listeNorme;

protected Sandbox coordinatorSandbox;

// made to report the number of object processed by the phase
private int reportNumberOfObject = 0;
Expand Down Expand Up @@ -98,6 +101,8 @@ protected ApiService(String aCurrentPhase, String aParametersEnvironment, String
LoggerHelper.error(LOGGER_APISERVICE, ApiService.class, "Error in initializing connexion");
}

this.coordinatorSandbox = new Sandbox(this.connexion.getCoordinatorConnection(), this.envExecution);

this.setParamBatch(paramBatch);

// Initialisation de la phase
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package fr.insee.arc.core.service.global.bo;

import java.sql.Connection;

public class Sandbox {

private Connection connection;

private String schema;


/**
* instanciate a sandbox reference for execution
* @param connection
* @param schema
*/
public Sandbox(Connection connection, String schema) {
super();
this.connection = connection;
this.schema = schema;
}

public Connection getConnection() {
return connection;
}

public String getSchema() {
return schema;
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package fr.insee.arc.core.service.global.dao;

import java.io.File;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collector;
import java.util.stream.Collectors;

import fr.insee.arc.core.dataobjects.ArcPreparedStatementBuilder;
import fr.insee.arc.core.dataobjects.ColumnEnum;
import fr.insee.arc.core.dataobjects.DataObjectService;
import fr.insee.arc.core.dataobjects.ViewEnum;
import fr.insee.arc.utils.dao.CopyObjectsToDatabase;
import fr.insee.arc.utils.dao.SQL;
import fr.insee.arc.utils.dao.UtilitaireDao;
import fr.insee.arc.utils.exception.ArcException;
import fr.insee.arc.utils.structure.GenericBean;
import fr.insee.arc.utils.utils.FormatSQL;

public class DataStorage {

private DataStorage() {
throw new IllegalStateException("dao class");
}

/**
* retrieve the data storage identifiers registered in database
*
* @return
* @throws ArcException
*/
public static List<String> execQuerySelectDatastorage(Connection connection) throws ArcException {
ArcPreparedStatementBuilder query = new ArcPreparedStatementBuilder();
query.build(SQL.SELECT, ColumnEnum.ID_ENTREPOT, SQL.FROM, ViewEnum.IHM_ENTREPOT.getFullName());
return new GenericBean(UtilitaireDao.get(0).executeRequest(connection, query))
.getColumnValues(ColumnEnum.ID_ENTREPOT.getColumnName());
}

/**
* register a list of files in a table
*
* @param fichiers
* @return
* @throws ArcException
*/
public static void execQueryRegisterFilesInDatabase(Connection connection, List<File> files) throws ArcException {
List<String> filenames = files.stream().filter(e -> !e.isDirectory()).map(e -> e.getName())
.collect(Collectors.toList());

CopyObjectsToDatabase.execCopyFromGenericBean(connection, ViewEnum.TMP_FILES.getTableName(), new GenericBean(
ColumnEnum.FILE_NAME.getColumnName(), ColumnEnum.FILE_NAME.getColumnType().getTypeName(), filenames));

}

/**
* given a list of files, return the list of files with no trace in the
* pilotage_archive table
*
* @param connection
* @param sanbox
* @return
* @throws ArcException
*/
public static List<String> execQuerySelectFilesNotInRegisteredArchives(Connection connection, String sanbox)
throws ArcException {
ArcPreparedStatementBuilder query = new ArcPreparedStatementBuilder();
query.build(SQL.SELECT, ColumnEnum.FILE_NAME, SQL.FROM, ViewEnum.TMP_FILES, SQL.AS, ViewEnum.ALIAS_A);
query.build(SQL.WHERE, SQL.NOT, SQL.EXISTS);
query.build("(");
query.build(SQL.SELECT, SQL.FROM, ViewEnum.PILOTAGE_ARCHIVE.getFullName(sanbox), SQL.AS, ViewEnum.ALIAS_B);
query.build(SQL.WHERE, ColumnEnum.NOM_ARCHIVE.alias(ViewEnum.ALIAS_B), "=",
ColumnEnum.FILE_NAME.alias(ViewEnum.ALIAS_A));
query.build(")");

return new GenericBean(UtilitaireDao.get(0).executeRequest(connection, query))
.getColumnValues(ColumnEnum.FILE_NAME.getColumnName());
}

}
Original file line number Diff line number Diff line change
@@ -1,9 +1,19 @@
package fr.insee.arc.core.service.global.dao;

import java.io.File;
import java.io.IOException;
import java.util.List;

import org.apache.commons.io.FileUtils;

import fr.insee.arc.core.dataobjects.ArcPreparedStatementBuilder;
import fr.insee.arc.core.model.TraitementEtat;
import fr.insee.arc.core.model.TraitementPhase;
import fr.insee.arc.utils.dao.UtilitaireDao;
import fr.insee.arc.utils.exception.ArcException;
import fr.insee.arc.utils.exception.ArcExceptionMessage;
import fr.insee.arc.utils.files.FileUtilsArc;
import fr.insee.arc.utils.structure.GenericBean;

public class FileSystemManagement {

Expand Down Expand Up @@ -65,4 +75,21 @@ public static String directoryPhaseEtatEnCours(String rootDirectory, String env,
return directoryPhaseEtat(rootDirectory, env, t, TraitementEtat.ENCOURS);
}

/**
* delete a file if it is the same as another reference file
* @param fileToDeleteIfSame
* @param fileToCompare
* @throws ArcException
*/
public static void deleteFileIfSameAs(File fileToDeleteIfSame, File fileToCompare) throws ArcException
{
try {
if (fileToCompare.exists() && FileUtils.contentEquals(fileToCompare, fileToDeleteIfSame)) {
FileUtilsArc.delete(fileToDeleteIfSame);
}
} catch (IOException exception) {
throw new ArcException(exception, ArcExceptionMessage.FILE_DELETE_FAILED, fileToDeleteIfSame);
}
}

}
Loading

0 comments on commit d0453cc

Please sign in to comment.