Skip to content

Commit

Permalink
Merge 'Create ReproVIP Directory' and 'Create Structured JSON' functi…
Browse files Browse the repository at this point in the history 
…onalities into a single user action
  • Loading branch information
@alexcornier
alexcornier committed Oct 24, 2023
1 parent 741b0d6 commit 2dc0177
Show file tree
Hide file tree
Showing 9 changed files with 73 additions and 174 deletions.
4 changes: 2 additions & 2 deletions vip-api/src/main/java/fr/insalyon/creatis/vip/api/business/ExecutionBusiness.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ public Execution getExecution(String executionId, boolean summarize)

// Outputs
List<InOutData> outputs = workflowBusiness.getOutputData(
executionId, currentUserProvider.get().getFolder(), false);
executionId, currentUserProvider.get().getFolder());
for (InOutData iod : outputs) {
if (!e.getReturnedFiles().containsKey(iod.getProcessor())) {
e.getReturnedFiles().put(iod.getProcessor(), new ArrayList<>());
Expand Down Expand Up @@ -439,7 +439,7 @@ public List<PathProperties> getExecutionResultsPaths(String executionId)
List<InOutData> outputs;
try {
outputs = workflowBusiness.getOutputData(
executionId, currentUserProvider.get().getFolder(), false);
executionId, currentUserProvider.get().getFolder());
} catch (BusinessException e) {
throw new ApiException(e);
}
Expand Down
4 changes: 1 addition & 3 deletions ...ication/src/main/java/fr/insalyon/creatis/vip/application/client/rpc/ReproVipService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,6 @@ public static ReproVipServiceAsync getInstance() {
public void executionAdminEmail(Execution execution);
void addExecution(Execution execution) throws CoreException;
void updateExecution(String executionID, String newStatus) throws CoreException;
void executionOutputData(String executionID) throws CoreException;
String downloadJsonOutputData(String executionName, String executionID, String version) throws CoreException;
void createReproVipDirectory(String executionName, String executionID, String version) throws CoreException;
String createReproVipDirectory(String executionName, String executionID, String version) throws CoreException;
}

4 changes: 1 addition & 3 deletions ...on/src/main/java/fr/insalyon/creatis/vip/application/client/rpc/ReproVipServiceAsync.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ public interface ReproVipServiceAsync {
void executionAdminEmail(Execution execution, AsyncCallback<Void> callback);
void addExecution(Execution execution, AsyncCallback<Void> asyncCallback);
void updateExecution(String executionID, String newStatus, AsyncCallback<Void> asyncCallback);
void executionOutputData(String executionID, AsyncCallback<Void> callback);
void downloadJsonOutputData(String executionName, String executionID, String version, AsyncCallback<String> callback);
void createReproVipDirectory(String executionName, String execution, String version, AsyncCallback<Void> callback);
void createReproVipDirectory(String executionName, String execution, String version, AsyncCallback<String> callback);
}

34 changes: 5 additions & 29 deletions ...nsalyon/creatis/vip/application/client/view/system/application/ExecutionsContextMenu.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,20 +32,10 @@ public ExecutionsContextMenu(ModalWindow modal, String executionName, String exe
@Override
public void onClick(MenuItemClickEvent event) {
createReproVipDirectory(executionName, executionID, version);
//MakexExecutionPublic();
}
});

MenuItem DownloadOutputDataItem = new MenuItem("Donwload outputs");
DownloadOutputDataItem.setIcon(CoreConstants.ICON_INFO);
DownloadOutputDataItem.addClickHandler(new ClickHandler() {
@Override
public void onClick(MenuItemClickEvent event) {
DownloadOutputDataExecution(executionName, executionID, version);
}
});

this.setItems(OptionPublicExecutionItem, DownloadOutputDataItem);
this.setItems(OptionPublicExecutionItem);
}
private void MakexExecutionPublic() {

Check warning

Code scanning / PMD

Avoid unused private methods such as 'selectRandomEngine(String)'. Warning

Avoid unused private methods such as 'MakexExecutionPublic()'.
final AsyncCallback<Void> callback = new AsyncCallback<Void>() {
Expand All @@ -63,28 +53,14 @@ public void onSuccess(Void result) {
modal.show("Make execution public", true);
reproVipServiceAsync.updateExecution(executionID, "Public", callback);
}
private void DownloadOutputDataExecution(String executionName, String executionID, String version) {
final AsyncCallback<String> callback = new AsyncCallback<String>() {
@Override
public void onFailure(Throwable caught) {
modal.hide();
SC.warn("Unable to download outputs of this execution public:<br />" + caught.getMessage());
}
@Override
public void onSuccess(String s) {
modal.hide();
new ViewerWindow("Execution Output Data", executionID, s).show();
}
};
modal.show("Download Outputs", true);
reproVipServiceAsync.downloadJsonOutputData(executionName, executionID, version, callback);
}
public void createReproVipDirectory(String executionName, String executionID, String version) {
reproVipServiceAsync.createReproVipDirectory(executionName, executionID, version, new AsyncCallback<Void>() {
reproVipServiceAsync.createReproVipDirectory(executionName, executionID, version, new AsyncCallback<String>() {
public void onFailure(Throwable caught) {
SC.warn("Error creating ReproVip directory: " + caught.getMessage());
}
public void onSuccess(Void result) {

@Override
public void onSuccess(String s) {
SC.say("ReproVip directory successfully created");
}
});
Expand Down
153 changes: 48 additions & 105 deletions ...n/src/main/java/fr/insalyon/creatis/vip/application/server/business/ReproVipBusiness.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gwt.user.client.rpc.AsyncCallback;
import fr.insalyon.creatis.vip.application.client.bean.AppVersion;
import fr.insalyon.creatis.vip.application.client.bean.InOutData;
import fr.insalyon.creatis.vip.application.client.bean.Job;
import fr.insalyon.creatis.vip.application.client.bean.Task;
Expand All @@ -25,11 +26,15 @@

import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;

@Service
@Transactional
Expand All @@ -40,6 +45,8 @@ public class ReproVipBusiness {
@Autowired
private WorkflowBusiness workflowBusiness;
@Autowired
private ApplicationBusiness applicationBusiness;
@Autowired
private EmailBusiness emailBusiness;
@Autowired
private ExecutionInOutData executionInOutData;
Expand Down Expand Up @@ -80,7 +87,7 @@ public void executionAdminEmail(Execution execution) throws DAOException, Busine
public ExecutionInOutData executionOutputData(String executionID, User currentUser) throws ApplicationException, BusinessException {
logger.info("Fetching data for executionID: {}", executionID);

List<InOutData> outputData = workflowBusiness.getOutputData(executionID, currentUser.getFolder(), false);
List<InOutData> outputData = workflowBusiness.getOutputData(executionID, currentUser.getFolder());
logger.info(String.valueOf(outputData));
List<InOutData> inputData = workflowBusiness.getInputData(executionID, currentUser.getFolder());

Expand Down Expand Up @@ -109,9 +116,9 @@ public ExecutionJobTaskData getExecutionJobTaskData(String executionID) throws B
}
return new ExecutionJobTaskData(jobList);
}
public String createJsonOutputData(String executionName, String executionID, String version, User currentUser)
throws ApplicationException, BusinessException {
List<String> filesToDownload = getFilesToCopyPaths(executionName, executionID, version, currentUser);
public String generateReprovipJson(Path reproVipDir, String executionName, String executionID, String version, User currentUser)
throws BusinessException {
List<String> filesToDownload = getFilesToCopyPaths(executionName, executionID, version);

Map<String, Object> structuredJson = new HashMap<>();

Expand Down Expand Up @@ -140,141 +147,77 @@ public String createJsonOutputData(String executionName, String executionID, Str
String json = objectMapper.writeValueAsString(structuredJson);
logger.info(json);

String filePath = "/vip/ReproVip/structuredOutput.json";
saveJsonToFile(json, filePath);
Path reprovipJsonPath = reproVipDir.resolve("structuredOutput.json");
saveJsonToFile(json, reprovipJsonPath);

return json;
} catch (JsonProcessingException e) {
throw new ApplicationException(ApplicationException.ApplicationError.valueOf("Failed to convert structured output to JSON"), e);
} catch (IOException e) {
throw new BusinessException("Failed to save JSON to file", e);
}
}
public void saveJsonToFile(String jsonContent, String filePath) throws IOException {
try (FileWriter fileWriter = new FileWriter(filePath);
public void saveJsonToFile(String jsonContent, Path filePath) throws IOException {
Files.writeString(filePath, jsonContent);
/*try (FileWriter fileWriter = new FileWriter(filePath.toFile());
BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) {
bufferedWriter.write(jsonContent);
}
}*/
}
public void createReproVipDirectory(String executionName, String executionID, String version, User currentUser) {
public String createReproVipDirectory(String executionName, String executionID, String version, User currentUser) throws BusinessException {
Path reproVipDir = Paths.get("/vip/ReproVip/" + executionID);
logger.info("Creating reprovip dir : {}", reproVipDir);
try {
logger.info("Attempting to create ReproVip directory...");
logger.info("Version de l'app : " + version);

String reproVipDirPath = "/vip/ReproVip";
File reproVipDir = new File(reproVipDirPath);

if (!reproVipDir.exists()) {
if (reproVipDir.mkdirs()) {
logger.info("ReproVip directory successfully created");
} else {
logger.error("Error creating ReproVip directory");
}
}

List<InOutData> outputData = workflowBusiness.getOutputData(executionID, currentUser.getFolder(), true);
logger.info(outputData.get(0).getPath());

if (outputData != null && !outputData.isEmpty()) {
String outputPath = "/vip/grida/downloads" + outputData.get(0).getPath();
if (outputPath != null) {
File outputFile = new File(outputPath);
if (outputFile.exists()) {
Files.copy(outputFile.toPath(), new File(reproVipDir, outputFile.getName()).toPath(), StandardCopyOption.REPLACE_EXISTING);
logger.info("Output file {} successfully copied to ReproVip directory", outputPath);
} else {
logger.warn("Output file {} not found", outputPath);
}
} else {
logger.warn("No output path found for executionID: {}", executionID);
}
} else {
logger.warn("No output data found for executionID: {}", executionID);
}

String workflowPath = String.valueOf(workflowBusiness.getRawApplicationDescriptorPath(currentUser, executionName, version));
logger.info(workflowPath);
File workflowFile = new File(workflowPath);
if (workflowFile.exists()) {
Files.copy(workflowFile.toPath(), new File(reproVipDir, workflowFile.getName()).toPath(), StandardCopyOption.REPLACE_EXISTING);
logger.info("Workflow file successfully copied to ReproVip directory");
} else {
logger.warn("Workflow file not found");
if ( ! Files.exists(reproVipDir)) {
Files.createDirectories(reproVipDir);
}
} catch (IOException e) {
logger.error("Exception creating the a reprovip directory {}", reproVipDir, e);
throw new RuntimeException(e);
}
copyProvenanceFiles(reproVipDir, executionID);
return generateReprovipJson(reproVipDir, executionName, executionID, version, currentUser);
}

logger.info("Workflows path: " + server.getWorkflowsPath());
public void copyProvenanceFiles(Path reproVipDir, String executionID) {
try {
logger.debug("Workflows path: " + server.getWorkflowsPath());

String provenanceDirPath = server.getWorkflowsPath() + "/" + executionID + "/provenance";
File provenanceDir = new File(provenanceDirPath);
if (!provenanceDir.exists()) {
logger.warn("Provenance directory does not exist: " + provenanceDirPath);
Path provenanceDirPath = Paths.get(server.getWorkflowsPath() + "/" + executionID + "/provenance");
if ( ! Files.exists(provenanceDirPath)) {
logger.error("Provenance directory does not exist: " + provenanceDirPath);
return;
}

File[] allFiles = provenanceDir.listFiles();
if (allFiles != null) {
for (File file : allFiles) {
logger.info("Found file in provenance directory: " + file.getName());
}
}
FilenameFilter filter = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.endsWith(".sh.provenance.json");
}
};

File[] matchingFiles = provenanceDir.listFiles(filter);

if (matchingFiles != null && matchingFiles.length > 0) {
File provenanceFile = matchingFiles[0];
if (provenanceFile.exists()) {
Files.copy(provenanceFile.toPath(), new File(reproVipDir, provenanceFile.getName()).toPath(), StandardCopyOption.REPLACE_EXISTING);
logger.info(provenanceFile.getName() + " file successfully copied to ReproVip directory");
} else {
logger.warn(provenanceFile.getName() + " file not found");
try (Stream<Path> stream = Files.list(provenanceDirPath)) {
List<Path> provenanceFiles = stream
.filter(path -> path.toString().endsWith(".sh.provenance.json"))
.collect(Collectors.toList());
for (Path provenanceFile : provenanceFiles) {
Files.copy(provenanceFile, reproVipDir.resolve(provenanceFile.getFileName()), StandardCopyOption.REPLACE_EXISTING);
logger.info("{} file successfully copied to ReproVip directory", provenanceFile);
}
} else {
logger.warn("No matching provenance file found");
}

} catch (IOException e) {
logger.error("Exception while copying files to ReproVip directory", e);
throw new RuntimeException(e);
} catch (BusinessException e) {
logger.error("Exception creating the a reprovip directory {}", reproVipDir, e);
throw new RuntimeException(e);
}
}

public List<String> getFilesToCopyPaths(String executionName, String executionID, String version, User currentUser) throws BusinessException {
public List<String> getFilesToCopyPaths(String executionName, String executionID, String version) throws BusinessException {
List<String> paths = new ArrayList<>();

List<InOutData> outputData = workflowBusiness.getOutputData(executionID, currentUser.getFolder(), true);
List<InOutData> outputData = workflowBusiness.getRawOutputData(executionID);
if (outputData != null && !outputData.isEmpty()) {
String outputPath = outputData.get(0).getPath();
if (outputPath != null) {
paths.add(outputPath);
}
}

String workflowPath = String.valueOf(workflowBusiness.getRawApplicationDescriptorPath(currentUser, executionName, version));
if (workflowPath != null && !workflowPath.isEmpty()) {
paths.add(workflowPath);
}

String provenanceDirPath = server.getWorkflowsPath() + "/" + executionID + "/provenance";
File provenanceDir = new File(provenanceDirPath);
if (provenanceDir.exists()) {
FilenameFilter filter = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.endsWith(".sh.provenance.json");
}
};
File[] matchingFiles = provenanceDir.listFiles(filter);
if (matchingFiles != null && matchingFiles.length > 0) {
paths.add(matchingFiles[0].getAbsolutePath());
}
AppVersion appVersion = applicationBusiness.getVersion(executionName, version);
if (appVersion != null && appVersion.getJsonLfn() != null) {
paths.add(appVersion.getJsonLfn());
}

return paths;
Expand Down
17 changes: 12 additions & 5 deletions ...n/src/main/java/fr/insalyon/creatis/vip/application/server/business/WorkflowBusiness.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -564,7 +564,18 @@ public Simulation getSimulation(String simulationID, boolean refresh)
return simulation;
}


public List<InOutData> getOutputData(
String simulationID, String currentUserFolder) throws BusinessException {
return getOutputData(simulationID, currentUserFolder, false);
}

public List<InOutData> getRawOutputData(
String simulationID) throws BusinessException {
return getOutputData(simulationID, null, true);
}

private List<InOutData> getOutputData(
String simulationID, String currentUserFolder, boolean useRawPath)
throws BusinessException {

Expand All @@ -581,13 +592,9 @@ public List<InOutData> getOutputData(
list.add(new InOutData(path, output.getOutputID().getProcessor(),
output.getType().name()));
}
} catch (WorkflowsDBDAOException ex) {
} catch (WorkflowsDBDAOException | DataManagerException ex) {
logger.error("Error getting output data for {}", simulationID, ex);
throw new BusinessException(ex);
} catch (DataManagerException ex) {
if (!useRawPath) {
throw new BusinessException(ex);
}
}
return list;
}
Expand Down
1 change: 1 addition & 0 deletions ...ation/src/main/java/fr/insalyon/creatis/vip/application/server/dao/h2/SimulationData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -414,6 +414,7 @@ public Map<String, Integer> getNodeCountriesMap() throws DAOException {
+ "GROUP BY country ORDER BY country");
ps.setString(1, TaskStatus.COMPLETED.name());
ResultSet rs = ps.executeQuery();


Map<String, Integer> map = new HashMap<String, Integer>();
while (rs.next()) {
Expand Down
Loading

0 comments on commit 2dc0177

Please sign in to comment.