Release 3.11 (PR #81)

virtual-imaging-platform · Oct 23, 2024 · b1178ce · b1178ce
2 parents 59f830e + a8c981e
commit b1178ce
Show file tree

Hide file tree

Showing 17 changed files with 1,649 additions and 90 deletions.
diff --git a/pom.xml b/pom.xml
@@ -1,11 +1,13 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0"
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
     <modelVersion>4.0.0</modelVersion>
+
     <groupId>fr.insalyon.creatis</groupId>
     <artifactId>gasw</artifactId>
     <packaging>jar</packaging>
-    <version>3.10.0</version>
+    <version>3.11.0</version>
     <name>GASW</name>
 
     <properties>
@@ -73,7 +75,6 @@
     </build>
 
     <dependencies>
-
         <dependency>
             <groupId>log4j</groupId>
             <artifactId>log4j</artifactId>
@@ -116,7 +117,6 @@
           <version>5.2.0</version>
           <scope>test</scope>
         </dependency>
-
     </dependencies>
 
     <repositories>
@@ -139,7 +139,6 @@
             <name>Internal Snapshots</name>
             <url>https://vip.creatis.insa-lyon.fr:9007/nexus/repository/snapshots</url>
         </snapshotRepository>
-
     </distributionManagement>
 
-</project>
+</project>
diff --git a/src/main/java/fr/insalyon/creatis/gasw/GaswConfiguration.java b/src/main/java/fr/insalyon/creatis/gasw/GaswConfiguration.java
@@ -32,24 +32,14 @@
  */
 package fr.insalyon.creatis.gasw;
 
-import fr.insalyon.creatis.gasw.bean.SEEntryPoint;
-import fr.insalyon.creatis.gasw.bean.SEEntryPointID;
-import fr.insalyon.creatis.gasw.dao.DAOException;
-import fr.insalyon.creatis.gasw.dao.DAOFactory;
-import fr.insalyon.creatis.gasw.plugin.DatabasePlugin;
-import fr.insalyon.creatis.gasw.plugin.ExecutorPlugin;
-import fr.insalyon.creatis.gasw.plugin.ListenerPlugin;
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.List;
-import net.xeoh.plugins.base.PluginManager;
-import net.xeoh.plugins.base.impl.PluginManagerFactory;
-import net.xeoh.plugins.base.util.JSPFProperties;
-import net.xeoh.plugins.base.util.PluginManagerUtil;
+
 import org.apache.commons.configuration.ConfigurationException;
 import org.apache.commons.configuration.PropertiesConfiguration;
 import org.apache.log4j.Logger;
@@ -58,6 +48,18 @@
 import org.hibernate.service.ServiceRegistry;
 import org.hibernate.service.ServiceRegistryBuilder;
 
+import fr.insalyon.creatis.gasw.bean.SEEntryPoint;
+import fr.insalyon.creatis.gasw.bean.SEEntryPointID;
+import fr.insalyon.creatis.gasw.dao.DAOException;
+import fr.insalyon.creatis.gasw.dao.DAOFactory;
+import fr.insalyon.creatis.gasw.plugin.DatabasePlugin;
+import fr.insalyon.creatis.gasw.plugin.ExecutorPlugin;
+import fr.insalyon.creatis.gasw.plugin.ListenerPlugin;
+import net.xeoh.plugins.base.PluginManager;
+import net.xeoh.plugins.base.impl.PluginManagerFactory;
+import net.xeoh.plugins.base.util.JSPFProperties;
+import net.xeoh.plugins.base.util.PluginManagerUtil;
+
 /**
  *
  * @author Rafael Ferreira da Silva
@@ -88,9 +90,11 @@ public class GaswConfiguration {
     private String voUseCloseSE;
     // Boutiques installation
     private String boshCVMFSPath;
+    private String apptainerPath;
     private String containersCVMFSPath;
     private String udockerTag;
     private String boutiquesProvenanceDir;
+    private String boutiquesFileName;
     // Failover Server
     private boolean failOverEnabled;
     private String failOverHost;
@@ -161,9 +165,11 @@ private void loadConfigurationFile() throws GaswException {
             voUseCloseSE = config.getString(GaswConstants.LAB_VO_USE_CLOSE_SE, "\"true\"");
 
             boshCVMFSPath = config.getString(GaswConstants.LAB_BOSH_CVMFS_PATH, "\"/cvmfs/biomed.egi.eu/vip/virtualenv/bin\"");
+            apptainerPath = config.getString(GaswConstants.LAB_APPTAINER_PATH, "\"/cvmfs/dirac.egi.eu/container/apptainer/bin\"");
             containersCVMFSPath = config.getString(GaswConstants.LAB_CONTAINERS_CVMFS_PATH, "\"/cvmfs/biomed.egi.eu/vip/udocker/containers\"");
             udockerTag = config.getString(GaswConstants.LAB_UDOCKER_TAG, "\"1.3.1\"");
             boutiquesProvenanceDir = config.getString(GaswConstants.LAB_BOUTIQUES_PROV_DIR, "\"$HOME/.cache/boutiques/data\"");
+            boutiquesFileName = config.getString(GaswConstants.LAB_BOUTIQUES_FILE_NAME, "workflow.json");
 
             failOverEnabled = config.getBoolean(GaswConstants.LAB_FAILOVER_ENABLED, false);
             failOverHost = config.getString(GaswConstants.LAB_FAILOVER_HOST, "localhost");
@@ -194,9 +200,11 @@ private void loadConfigurationFile() throws GaswException {
             config.setProperty(GaswConstants.LAB_VO_USE_CLOSE_SE, voUseCloseSE);
 
 	        config.setProperty(GaswConstants.LAB_BOSH_CVMFS_PATH, boshCVMFSPath);
+            config.setProperty(GaswConstants.LAB_APPTAINER_PATH, apptainerPath);
             config.setProperty(GaswConstants.LAB_CONTAINERS_CVMFS_PATH, containersCVMFSPath);
             config.setProperty(GaswConstants.LAB_UDOCKER_TAG, udockerTag);
             config.setProperty(GaswConstants.LAB_BOUTIQUES_PROV_DIR, boutiquesProvenanceDir);
+            config.setProperty(GaswConstants.LAB_BOUTIQUES_FILE_NAME, boutiquesFileName);
 
             config.setProperty(GaswConstants.LAB_FAILOVER_ENABLED, failOverEnabled);
             config.setProperty(GaswConstants.LAB_FAILOVER_HOST, failOverHost);
@@ -417,10 +425,18 @@ public String getBoshCVMFSPath() {
         return boshCVMFSPath;
     }
 
+    public String getApptainerPath() {
+        return apptainerPath;
+    }
+
     public String getBoutiquesProvenanceDir() {
         return boutiquesProvenanceDir;
     }
 
+    public String getBoutiquesFilename() {
+        return boutiquesFileName;
+    }
+
     public String getContainersCVMFSPath() {
         return containersCVMFSPath;
     }
@@ -460,4 +476,4 @@ public int getMinAvgDownloadThroughput() {
     public int getDefaultRetryCount() {
         return defaultRetryCount;
     }
-}
+}
diff --git a/src/main/java/fr/insalyon/creatis/gasw/GaswConstants.java b/src/main/java/fr/insalyon/creatis/gasw/GaswConstants.java
@@ -60,9 +60,11 @@ public class GaswConstants {
     public static final String LAB_VO_NAME = "vo.name";
     public static final String LAB_VO_USE_CLOSE_SE = "vo.use.close.se";
     public static final String LAB_BOSH_CVMFS_PATH = "bosh.cvmfs.path";
+    public static final String LAB_APPTAINER_PATH = "apptainer.path";
     public static final String LAB_CONTAINERS_CVMFS_PATH = "containers.cvmfs.path";
     public static final String LAB_UDOCKER_TAG = "udocker.tag";
     public static final String LAB_BOUTIQUES_PROV_DIR = "boutiques.provenance.dir";
+    public static final String LAB_BOUTIQUES_FILE_NAME = "boutiques.filename";
     // timeouts used in lcg-c*
     //public static final int SEND_RECEIVE_TIMEOUT = 900;
     public static final String LAB_MIN_AVG_DOWNLOAD_THROUGHPUT = "min.avg.download.throughput";
@@ -86,4 +88,8 @@ public class GaswConstants {
     public static final String PROVENANCE_EXT = ".provenance.json";
     // Environment Variables
     public static final String ENV_EXECUTOR = "executor";
+    // moteur-lite constants
+    public static final int numberOfReplicas = 1;
+    public static final String INVOCATION_DIR = "./inv";
+    public static final String CONFIG_DIR = "./config";
 }
diff --git a/src/main/java/fr/insalyon/creatis/gasw/GaswInput.java b/src/main/java/fr/insalyon/creatis/gasw/GaswInput.java
@@ -33,6 +33,8 @@
 package fr.insalyon.creatis.gasw;
 
 import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -46,16 +48,23 @@ public class GaswInput {
     private List<String> parameters;
     private List<URI> downloads;
     private List<GaswUpload> uploads;
+    private URI uploadURI;
     private Map<String, String> gaswVariables;
     private Map<String, String> envVariables;
+    private String invocationString;
+    private String jobId;
+    private String applicationName;
+    private Boolean moteurLiteEnabled = false;
 
     /**
-     *
      * @param executableName Name of the executable file.
-     * @param parameters List of parameters associated with the command.
-     * @param downloads List of input files to be downloaded in the worker node.
-     * @param uploads List of output files to be uploaded to a Storage Element.
+     * @param parameters     List of parameters associated with the command.
+     * @param downloads      List of input files to be downloaded in the worker node.
+     * @param uploads        List of output files to be uploaded to a Storage Element.
+     * @param gaswVariables  Map of GASW variables.
+     * @param envVariables   Map of environment variables.
      */
+
     public GaswInput(String executableName, List<String> parameters,
             List<URI> downloads, List<GaswUpload> uploads,
             Map<String, String> gaswVariables, Map<String, String> envVariables) {
@@ -68,6 +77,30 @@ public GaswInput(String executableName, List<String> parameters,
         this.envVariables = envVariables;
     }
 
+ /**
+ * @param applicationName Name of the application.
+ * @param executableName Name of the executable file.
+ * @param downloads List of input files to be downloaded in the worker node.
+ * @param uploadURI URI for the output file upload destination.
+ * @param invocationString String representation of the invocation.
+ * @param jobId Job ID.
+ */
+
+    public GaswInput(String applicationName, String executableName, List<URI> downloads,
+            URI uploadURI, String invocationString, String jobId) {
+
+        this.executableName = executableName;
+        this.downloads = downloads;
+        this.uploadURI = uploadURI;
+        this.invocationString = invocationString;
+        this.jobId = jobId;
+        this.applicationName = applicationName;
+        this.moteurLiteEnabled = true;
+        this.parameters = new ArrayList<>();
+        this.gaswVariables = new HashMap<>();
+        this.envVariables = new HashMap<>();
+    }
+
     /**
      * Adds a parameter to the list of parameters.
      *
@@ -95,6 +128,10 @@ public void addUpload(GaswUpload upload) {
         this.uploads.add(upload);
     }
 
+    public void setUploadURI(URI uploadURI) {
+        this.uploadURI = uploadURI;
+    }
+
     /**
      * Adds a GASW variable to the request.
      *
@@ -127,6 +164,10 @@ public List<GaswUpload> getUploads() {
         return uploads;
     }
 
+    public URI getUploadURI() {
+        return uploadURI;
+    }
+
     public String getExecutableName() {
         return executableName;
     }
@@ -138,4 +179,20 @@ public String getGaswVariable(String key) {
     public Map<String, String> getEnvVariables() {
         return envVariables;
     }
-}
+
+    public String getInvocationString() {
+        return invocationString;
+    }
+
+    public String getJobId() {
+        return jobId;
+    }
+
+    public String getApplicationName(){
+        return applicationName;
+    }
+
+    public Boolean isMoteurLiteEnabled(){
+        return moteurLiteEnabled;
+    }
+}
diff --git a/src/main/java/fr/insalyon/creatis/gasw/GaswUtil.java b/src/main/java/fr/insalyon/creatis/gasw/GaswUtil.java
@@ -36,6 +36,7 @@
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.regex.Pattern;
+
 import org.apache.log4j.Logger;
 
 /**
@@ -84,8 +85,7 @@ public static Process getProcess(Logger logger, boolean redirectError,
         if (redirectError) {
             builder.redirectErrorStream(true);
         }
-
-        builder.environment().put("X509_USER_PROXY", System.getenv("X509_USER_PROXY"));
+
         return builder.start();
     }
 
@@ -117,4 +117,4 @@ public static void closeProcess(Process process) throws IOException {
     public static boolean isUri(String s) {
         return uriPattern.matcher(s).find();
     }
-}
+}