Skip to content

Commit

Permalink
feat: new parameters KUBERNETES_EXECUTOR_EPHEMERAL for volatile db
Browse files Browse the repository at this point in the history
  • Loading branch information
Nolife999 committed Sep 20, 2024
1 parent 494fa63 commit f7afce2
Show file tree
Hide file tree
Showing 11 changed files with 23 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ fr.insee.arc.kubernetes.executor.database=${env.kubernetesExecutorDatabase:}
fr.insee.arc.kubernetes.executor.port=${env.kubernetesExecutorPort:}
fr.insee.arc.kubernetes.executor.cpu=${env.kubernetesExecutorCpu:}
fr.insee.arc.kubernetes.executor.ram=${env.kubernetesExecutorRam:}
fr.insee.arc.kubernetes.executor.ephemeral=${env.kubernetesExecutorEphemeral:}
fr.insee.arc.kubernetes.executor.volatile=${env.kubernetesExecutorVolatile:}

# Properties to activate export phase
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,7 @@ private JsonFileParameter() {
public static final String IMAGE = "{image}";
public static final String CPU = "{cpu}";
public static final String RAM = "{ram}";
public static final String EPHEMERAL = "{ephemeral}";


}
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ private static String[] kubernetesParameters(int executorReplicaIndex)
, JsonFileParameter.IMAGE, properties.getKubernetesExecutorImage()
, JsonFileParameter.CPU, properties.getKubernetesExecutorCpu()
, JsonFileParameter.RAM, properties.getKubernetesExecutorRam()
, JsonFileParameter.EPHEMERAL, properties.getKubernetesExecutorEphemeral()
};
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -142,11 +142,13 @@ private ExecutorDatabaseStatefulTemplate() {
"resources": {
"requests": {
"cpu": "10m",
"memory": "16Mi"
"memory": "16Mi",
"ephemeral-storage": "{ephemeral}",
},
"limits": {
"cpu": "{cpu}",
"memory": "{ram}"
"memory": "{ram}",
"ephemeral-storage": "{ephemeral}",
}
},
"securityContext": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ public class PropertiesHandler {
private String kubernetesExecutorPort;
private String kubernetesExecutorCpu;
private String kubernetesExecutorRam;
private String kubernetesExecutorEphemeral;
private String kubernetesExecutorVolatile;

/* export data end of batch ? */
Expand Down Expand Up @@ -409,6 +410,14 @@ public String getKubernetesExecutorRam() {
public void setKubernetesExecutorRam(String kubernetesExecutorRam) {
this.kubernetesExecutorRam = kubernetesExecutorRam;
}

public String getKubernetesExecutorEphemeral() {
return kubernetesExecutorEphemeral;
}

public void setKubernetesExecutorEphemeral(String kubernetesExecutorEphemeral) {
this.kubernetesExecutorEphemeral = kubernetesExecutorEphemeral;
}

public String getKubernetesExecutorVolatile() {
return kubernetesExecutorVolatile;
Expand Down
1 change: 1 addition & 0 deletions arc-web/src/main/resources/applicationContext.xml
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
<property name="kubernetesExecutorPort" value="#{systemEnvironment['KUBERNETES_EXECUTOR_PORT'] ?: '${fr.insee.arc.kubernetes.executor.port}' }"></property>
<property name="kubernetesExecutorCpu" value="#{systemEnvironment['KUBERNETES_EXECUTOR_CPU'] ?: '${fr.insee.arc.kubernetes.executor.cpu}' }"></property>
<property name="kubernetesExecutorRam" value="#{systemEnvironment['KUBERNETES_EXECUTOR_RAM'] ?: '${fr.insee.arc.kubernetes.executor.ram}' }"></property>
<property name="kubernetesExecutorEphemeral" value="#{systemEnvironment['KUBERNETES_EXECUTOR_EPHEMERAL'] ?: '${fr.insee.arc.kubernetes.executor.ephemeral}' }"></property>
<property name="kubernetesExecutorVolatile" value="#{systemEnvironment['KUBERNETES_EXECUTOR_VOLATILE'] ?: '${fr.insee.arc.kubernetes.executor.volatile}' }"></property>
<property name="processExport" value="#{systemEnvironment['PROCESS_EXPORT'] ?: '${fr.insee.arc.process.export}' }"></property>

Expand Down
1 change: 1 addition & 0 deletions arc-web/src/main/resources/fr/insee/config/arc.properties
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ fr.insee.arc.kubernetes.executor.database=${env.kubernetesExecutorDatabase:}
fr.insee.arc.kubernetes.executor.port=${env.kubernetesExecutorPort:}
fr.insee.arc.kubernetes.executor.cpu=${env.kubernetesExecutorCpu:}
fr.insee.arc.kubernetes.executor.ram=${env.kubernetesExecutorRam:}
fr.insee.arc.kubernetes.executor.ephemeral=${env.kubernetesExecutorEphemeral:}
fr.insee.arc.kubernetes.executor.volatile=${env.kubernetesExecutorVolatile:}

# Properties to activate export phase
Expand Down
1 change: 1 addition & 0 deletions arc-ws/src/main/resources/fr/insee/config/arc.properties
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ fr.insee.arc.kubernetes.executor.database=${env.kubernetesExecutorDatabase:}
fr.insee.arc.kubernetes.executor.port=${env.kubernetesExecutorPort:}
fr.insee.arc.kubernetes.executor.cpu=${env.kubernetesExecutorCpu:}
fr.insee.arc.kubernetes.executor.ram=${env.kubernetesExecutorRam:}
fr.insee.arc.kubernetes.executor.ephemeral=${env.kubernetesExecutorEphemeral:}
fr.insee.arc.kubernetes.executor.volatile=${env.kubernetesExecutorVolatile:}

# Properties to activate export phase
Expand Down
1 change: 1 addition & 0 deletions arc-ws/src/main/webapp/WEB-INF/applicationContext.xml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
<property name="kubernetesExecutorPort" value="#{systemEnvironment['KUBERNETES_EXECUTOR_PORT'] ?: '${fr.insee.arc.kubernetes.executor.port}' }"></property>
<property name="kubernetesExecutorCpu" value="#{systemEnvironment['KUBERNETES_EXECUTOR_CPU'] ?: '${fr.insee.arc.kubernetes.executor.cpu}' }"></property>
<property name="kubernetesExecutorRam" value="#{systemEnvironment['KUBERNETES_EXECUTOR_RAM'] ?: '${fr.insee.arc.kubernetes.executor.ram}' }"></property>
<property name="kubernetesExecutorEphemeral" value="#{systemEnvironment['KUBERNETES_EXECUTOR_EPHEMERAL'] ?: '${fr.insee.arc.kubernetes.executor.ephemeral}' }"></property>
<property name="kubernetesExecutorVolatile" value="#{systemEnvironment['KUBERNETES_EXECUTOR_VOLATILE'] ?: '${fr.insee.arc.kubernetes.executor.volatile}' }"></property>

<property name="processExport" value="#{systemEnvironment['PROCESS_EXPORT'] ?: '${fr.insee.arc.process.export}' }"></property>
Expand Down
2 changes: 1 addition & 1 deletion docker/script.sh
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,6 @@ echo "maven settings.xml set to $MAVEN_SETTINGS";
MAVEN_CONF="-s usr/src/app/$MAVEN_SETTINGS";
fi

mvn -f /usr/src/app/pom.xml clean package -DskipTests $MAVEN_CONF -Pdocker -Denv.logSettings=$LOG_SETTINGS -Denv.urlDatabase=$DATABASE_URL -Denv.usernameDatabase=$DATABASE_USER -Denv.passwordDatabase=$DATABASE_PASSWORD -Denv.restrictedUserDatabase=$DATABASE_RESTRICTED_USER -Denv.applicationDirectory=$APPLICATION_DIRECTORY -Denv.disableDebugGui=$DISABLE_DEBUG_GUI -Denv.kubernetesApiUri=$KUBERNETES_API_URI -Denv.kubernetesApiNamespace=KUBERNETES_API_NAMESPACE -Denv.kubernetesApiTokenPath=$KUBERNETES_API_TOKEN_PATH -Denv.kubernetesApiTokenValue=$KUBERNETES_API_TOKEN_VALUE -Denv.kubernetesExecutorImage=$KUBERNETES_EXECUTOR_IMAGE -Denv.kubernetesExecutorNumber=$KUBERNETES_EXECUTOR_NUMBER -Denv.kubernetesExecutorLabel=$KUBERNETES_EXECUTOR_LABEL -Denv.kubernetesExecutorUser=$KUBERNETES_EXECUTOR_USER -Denv.kubernetesExecutorDatabase=$KUBERNETES_EXECUTOR_DATABASE -Denv.kubernetesExecutorPort=$KUBERNETES_EXECUTOR_PORT -Denv.kubernetesExecutorCpu=$KUBERNETES_EXECUTOR_CPU -Denv.kubernetesExecutorRam=$KUBERNETES_EXECUTOR_RAM -Denv.kubernetesExecutorVolatile=$KUBERNETES_EXECUTOR_VOLATILE -Denv.processExport=$PROCESS_EXPORT -Denv.s3InputApiUri=$S3_INPUT_API_URI -Denv.s3InputBucket=$S3_INPUT_BUCKET -Denv.s3InputAccess=$S3_INPUT_ACCESS -Denv.s3InputSecret=$S3_INPUT_SECRET -Denv.s3OutputApiUri=$S3_OUTPUT_API_URI -Denv.s3OutputBucket=$S3_OUTPUT_BUCKET -Denv.s3OutputAccess=$S3_OUTPUT_ACCESS -Denv.s3OutputSecret=$S3_OUTPUT_SECRET -Denv.s3OutputParquetKey=$S3_OUTPUT_PARQUET_KEY -Denv.envExecution=$ENV_EXECUTION -Dfr.insee.arc.roles.admin=$KEYCLOAK_AUTHORIZED_ROLES -Dfr.insee.keycloak.realm=$KEYCLOAK_REALM -Dfr.insee.keycloak.server=$KEYCLOAK_SERVER -Dfr.insee.keycloak.resource=$KEYCLOAK_RESOURCE -Dfr.insee.keycloak.credentials.secret=$KEYCLOAK_CREDENTIALS -Dfr.insee.arc.files.retention.days=FILES_RETENTION_DAYS;
mvn -f /usr/src/app/pom.xml clean package -DskipTests $MAVEN_CONF -Pdocker -Denv.logSettings=$LOG_SETTINGS -Denv.urlDatabase=$DATABASE_URL -Denv.usernameDatabase=$DATABASE_USER -Denv.passwordDatabase=$DATABASE_PASSWORD -Denv.restrictedUserDatabase=$DATABASE_RESTRICTED_USER -Denv.applicationDirectory=$APPLICATION_DIRECTORY -Denv.disableDebugGui=$DISABLE_DEBUG_GUI -Denv.kubernetesApiUri=$KUBERNETES_API_URI -Denv.kubernetesApiNamespace=KUBERNETES_API_NAMESPACE -Denv.kubernetesApiTokenPath=$KUBERNETES_API_TOKEN_PATH -Denv.kubernetesApiTokenValue=$KUBERNETES_API_TOKEN_VALUE -Denv.kubernetesExecutorImage=$KUBERNETES_EXECUTOR_IMAGE -Denv.kubernetesExecutorNumber=$KUBERNETES_EXECUTOR_NUMBER -Denv.kubernetesExecutorLabel=$KUBERNETES_EXECUTOR_LABEL -Denv.kubernetesExecutorUser=$KUBERNETES_EXECUTOR_USER -Denv.kubernetesExecutorDatabase=$KUBERNETES_EXECUTOR_DATABASE -Denv.kubernetesExecutorPort=$KUBERNETES_EXECUTOR_PORT -Denv.kubernetesExecutorCpu=$KUBERNETES_EXECUTOR_CPU -Denv.kubernetesExecutorRam=$KUBERNETES_EXECUTOR_RAM -Denv.kubernetesExecutorEphemeral=$KUBERNETES_EXECUTOR_EPHEMERAL -Denv.kubernetesExecutorVolatile=$KUBERNETES_EXECUTOR_VOLATILE -Denv.processExport=$PROCESS_EXPORT -Denv.s3InputApiUri=$S3_INPUT_API_URI -Denv.s3InputBucket=$S3_INPUT_BUCKET -Denv.s3InputAccess=$S3_INPUT_ACCESS -Denv.s3InputSecret=$S3_INPUT_SECRET -Denv.s3OutputApiUri=$S3_OUTPUT_API_URI -Denv.s3OutputBucket=$S3_OUTPUT_BUCKET -Denv.s3OutputAccess=$S3_OUTPUT_ACCESS -Denv.s3OutputSecret=$S3_OUTPUT_SECRET -Denv.s3OutputParquetKey=$S3_OUTPUT_PARQUET_KEY -Denv.envExecution=$ENV_EXECUTION -Dfr.insee.arc.roles.admin=$KEYCLOAK_AUTHORIZED_ROLES -Dfr.insee.keycloak.realm=$KEYCLOAK_REALM -Dfr.insee.keycloak.server=$KEYCLOAK_SERVER -Dfr.insee.keycloak.resource=$KEYCLOAK_RESOURCE -Dfr.insee.keycloak.credentials.secret=$KEYCLOAK_CREDENTIALS -Dfr.insee.arc.files.retention.days=FILES_RETENTION_DAYS;


1 change: 1 addition & 0 deletions user-guide/arc_parameters.md
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ ___
| kubernetesExecutorPort | KUBERNETES_EXECUTOR_PORT | fr.insee.arc.kubernetes.executor.port |
| kubernetesExecutorCpu | KUBERNETES_EXECUTOR_CPU | fr.insee.arc.kubernetes.executor.cpu |
| kubernetesExecutorRam | KUBERNETES_EXECUTOR_RAM | fr.insee.arc.kubernetes.executor.ram |
| kubernetesExecutorEphemeral | KUBERNETES_EXECUTOR_EPHEMERAL | fr.insee.arc.kubernetes.executor.ephemeral |
| kubernetesExecutorVolatile | KUBERNETES_EXECUTOR_VOLATILE | fr.insee.arc.kubernetes.executor.volatile |


Expand Down

0 comments on commit f7afce2

Please sign in to comment.