From 2dc01779f3a974146153eec3b83a4def3114c77c Mon Sep 17 00:00:00 2001 From: Alexandre Cornier Date: Tue, 24 Oct 2023 13:43:50 +0200 Subject: [PATCH] Merge 'Create ReproVIP Directory' and 'Create Structured JSON' functionalities into a single user action --- .../vip/api/business/ExecutionBusiness.java | 4 +- .../client/rpc/ReproVipService.java | 4 +- .../client/rpc/ReproVipServiceAsync.java | 4 +- .../application/ExecutionsContextMenu.java | 34 +--- .../server/business/ReproVipBusiness.java | 153 ++++++------------ .../server/business/WorkflowBusiness.java | 17 +- .../server/dao/h2/SimulationData.java | 1 + .../server/rpc/ReproVipServiceImpl.java | 28 +--- .../server/rpc/WorkflowServiceImpl.java | 2 +- 9 files changed, 73 insertions(+), 174 deletions(-) diff --git a/vip-api/src/main/java/fr/insalyon/creatis/vip/api/business/ExecutionBusiness.java b/vip-api/src/main/java/fr/insalyon/creatis/vip/api/business/ExecutionBusiness.java index 7ff03aedf..d296709c1 100644 --- a/vip-api/src/main/java/fr/insalyon/creatis/vip/api/business/ExecutionBusiness.java +++ b/vip-api/src/main/java/fr/insalyon/creatis/vip/api/business/ExecutionBusiness.java @@ -154,7 +154,7 @@ public Execution getExecution(String executionId, boolean summarize) // Outputs List outputs = workflowBusiness.getOutputData( - executionId, currentUserProvider.get().getFolder(), false); + executionId, currentUserProvider.get().getFolder()); for (InOutData iod : outputs) { if (!e.getReturnedFiles().containsKey(iod.getProcessor())) { e.getReturnedFiles().put(iod.getProcessor(), new ArrayList<>()); @@ -439,7 +439,7 @@ public List getExecutionResultsPaths(String executionId) List outputs; try { outputs = workflowBusiness.getOutputData( - executionId, currentUserProvider.get().getFolder(), false); + executionId, currentUserProvider.get().getFolder()); } catch (BusinessException e) { throw new ApiException(e); } diff --git a/vip-application/src/main/java/fr/insalyon/creatis/vip/application/client/rpc/ReproVipService.java b/vip-application/src/main/java/fr/insalyon/creatis/vip/application/client/rpc/ReproVipService.java index 1b7c0701b..ec7a70424 100644 --- a/vip-application/src/main/java/fr/insalyon/creatis/vip/application/client/rpc/ReproVipService.java +++ b/vip-application/src/main/java/fr/insalyon/creatis/vip/application/client/rpc/ReproVipService.java @@ -24,8 +24,6 @@ public static ReproVipServiceAsync getInstance() { public void executionAdminEmail(Execution execution); void addExecution(Execution execution) throws CoreException; void updateExecution(String executionID, String newStatus) throws CoreException; - void executionOutputData(String executionID) throws CoreException; - String downloadJsonOutputData(String executionName, String executionID, String version) throws CoreException; - void createReproVipDirectory(String executionName, String executionID, String version) throws CoreException; + String createReproVipDirectory(String executionName, String executionID, String version) throws CoreException; } diff --git a/vip-application/src/main/java/fr/insalyon/creatis/vip/application/client/rpc/ReproVipServiceAsync.java b/vip-application/src/main/java/fr/insalyon/creatis/vip/application/client/rpc/ReproVipServiceAsync.java index 0ac907bc6..300b88250 100644 --- a/vip-application/src/main/java/fr/insalyon/creatis/vip/application/client/rpc/ReproVipServiceAsync.java +++ b/vip-application/src/main/java/fr/insalyon/creatis/vip/application/client/rpc/ReproVipServiceAsync.java @@ -8,8 +8,6 @@ public interface ReproVipServiceAsync { void executionAdminEmail(Execution execution, AsyncCallback callback); void addExecution(Execution execution, AsyncCallback asyncCallback); void updateExecution(String executionID, String newStatus, AsyncCallback asyncCallback); - void executionOutputData(String executionID, AsyncCallback callback); - void downloadJsonOutputData(String executionName, String executionID, String version, AsyncCallback callback); - void createReproVipDirectory(String executionName, String execution, String version, AsyncCallback callback); + void createReproVipDirectory(String executionName, String execution, String version, AsyncCallback callback); } diff --git a/vip-application/src/main/java/fr/insalyon/creatis/vip/application/client/view/system/application/ExecutionsContextMenu.java b/vip-application/src/main/java/fr/insalyon/creatis/vip/application/client/view/system/application/ExecutionsContextMenu.java index 536835321..dca61c233 100644 --- a/vip-application/src/main/java/fr/insalyon/creatis/vip/application/client/view/system/application/ExecutionsContextMenu.java +++ b/vip-application/src/main/java/fr/insalyon/creatis/vip/application/client/view/system/application/ExecutionsContextMenu.java @@ -32,20 +32,10 @@ public ExecutionsContextMenu(ModalWindow modal, String executionName, String exe @Override public void onClick(MenuItemClickEvent event) { createReproVipDirectory(executionName, executionID, version); - //MakexExecutionPublic(); } }); - MenuItem DownloadOutputDataItem = new MenuItem("Donwload outputs"); - DownloadOutputDataItem.setIcon(CoreConstants.ICON_INFO); - DownloadOutputDataItem.addClickHandler(new ClickHandler() { - @Override - public void onClick(MenuItemClickEvent event) { - DownloadOutputDataExecution(executionName, executionID, version); - } - }); - - this.setItems(OptionPublicExecutionItem, DownloadOutputDataItem); + this.setItems(OptionPublicExecutionItem); } private void MakexExecutionPublic() { final AsyncCallback callback = new AsyncCallback() { @@ -63,28 +53,14 @@ public void onSuccess(Void result) { modal.show("Make execution public", true); reproVipServiceAsync.updateExecution(executionID, "Public", callback); } - private void DownloadOutputDataExecution(String executionName, String executionID, String version) { - final AsyncCallback callback = new AsyncCallback() { - @Override - public void onFailure(Throwable caught) { - modal.hide(); - SC.warn("Unable to download outputs of this execution public:
" + caught.getMessage()); - } - @Override - public void onSuccess(String s) { - modal.hide(); - new ViewerWindow("Execution Output Data", executionID, s).show(); - } - }; - modal.show("Download Outputs", true); - reproVipServiceAsync.downloadJsonOutputData(executionName, executionID, version, callback); - } public void createReproVipDirectory(String executionName, String executionID, String version) { - reproVipServiceAsync.createReproVipDirectory(executionName, executionID, version, new AsyncCallback() { + reproVipServiceAsync.createReproVipDirectory(executionName, executionID, version, new AsyncCallback() { public void onFailure(Throwable caught) { SC.warn("Error creating ReproVip directory: " + caught.getMessage()); } - public void onSuccess(Void result) { + + @Override + public void onSuccess(String s) { SC.say("ReproVip directory successfully created"); } }); diff --git a/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/business/ReproVipBusiness.java b/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/business/ReproVipBusiness.java index 36b1c8424..ccea216dc 100644 --- a/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/business/ReproVipBusiness.java +++ b/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/business/ReproVipBusiness.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gwt.user.client.rpc.AsyncCallback; +import fr.insalyon.creatis.vip.application.client.bean.AppVersion; import fr.insalyon.creatis.vip.application.client.bean.InOutData; import fr.insalyon.creatis.vip.application.client.bean.Job; import fr.insalyon.creatis.vip.application.client.bean.Task; @@ -25,11 +26,15 @@ import java.io.*; import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; @Service @Transactional @@ -40,6 +45,8 @@ public class ReproVipBusiness { @Autowired private WorkflowBusiness workflowBusiness; @Autowired + private ApplicationBusiness applicationBusiness; + @Autowired private EmailBusiness emailBusiness; @Autowired private ExecutionInOutData executionInOutData; @@ -80,7 +87,7 @@ public void executionAdminEmail(Execution execution) throws DAOException, Busine public ExecutionInOutData executionOutputData(String executionID, User currentUser) throws ApplicationException, BusinessException { logger.info("Fetching data for executionID: {}", executionID); - List outputData = workflowBusiness.getOutputData(executionID, currentUser.getFolder(), false); + List outputData = workflowBusiness.getOutputData(executionID, currentUser.getFolder()); logger.info(String.valueOf(outputData)); List inputData = workflowBusiness.getInputData(executionID, currentUser.getFolder()); @@ -109,9 +116,9 @@ public ExecutionJobTaskData getExecutionJobTaskData(String executionID) throws B } return new ExecutionJobTaskData(jobList); } - public String createJsonOutputData(String executionName, String executionID, String version, User currentUser) - throws ApplicationException, BusinessException { - List filesToDownload = getFilesToCopyPaths(executionName, executionID, version, currentUser); + public String generateReprovipJson(Path reproVipDir, String executionName, String executionID, String version, User currentUser) + throws BusinessException { + List filesToDownload = getFilesToCopyPaths(executionName, executionID, version); Map structuredJson = new HashMap<>(); @@ -140,116 +147,67 @@ public String createJsonOutputData(String executionName, String executionID, Str String json = objectMapper.writeValueAsString(structuredJson); logger.info(json); - String filePath = "/vip/ReproVip/structuredOutput.json"; - saveJsonToFile(json, filePath); + Path reprovipJsonPath = reproVipDir.resolve("structuredOutput.json"); + saveJsonToFile(json, reprovipJsonPath); return json; - } catch (JsonProcessingException e) { - throw new ApplicationException(ApplicationException.ApplicationError.valueOf("Failed to convert structured output to JSON"), e); } catch (IOException e) { throw new BusinessException("Failed to save JSON to file", e); } } - public void saveJsonToFile(String jsonContent, String filePath) throws IOException { - try (FileWriter fileWriter = new FileWriter(filePath); + public void saveJsonToFile(String jsonContent, Path filePath) throws IOException { + Files.writeString(filePath, jsonContent); + /*try (FileWriter fileWriter = new FileWriter(filePath.toFile()); BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) { bufferedWriter.write(jsonContent); - } + }*/ } - public void createReproVipDirectory(String executionName, String executionID, String version, User currentUser) { + public String createReproVipDirectory(String executionName, String executionID, String version, User currentUser) throws BusinessException { + Path reproVipDir = Paths.get("/vip/ReproVip/" + executionID); + logger.info("Creating reprovip dir : {}", reproVipDir); try { - logger.info("Attempting to create ReproVip directory..."); - logger.info("Version de l'app : " + version); - String reproVipDirPath = "/vip/ReproVip"; - File reproVipDir = new File(reproVipDirPath); - - if (!reproVipDir.exists()) { - if (reproVipDir.mkdirs()) { - logger.info("ReproVip directory successfully created"); - } else { - logger.error("Error creating ReproVip directory"); - } - } - - List outputData = workflowBusiness.getOutputData(executionID, currentUser.getFolder(), true); - logger.info(outputData.get(0).getPath()); - - if (outputData != null && !outputData.isEmpty()) { - String outputPath = "/vip/grida/downloads" + outputData.get(0).getPath(); - if (outputPath != null) { - File outputFile = new File(outputPath); - if (outputFile.exists()) { - Files.copy(outputFile.toPath(), new File(reproVipDir, outputFile.getName()).toPath(), StandardCopyOption.REPLACE_EXISTING); - logger.info("Output file {} successfully copied to ReproVip directory", outputPath); - } else { - logger.warn("Output file {} not found", outputPath); - } - } else { - logger.warn("No output path found for executionID: {}", executionID); - } - } else { - logger.warn("No output data found for executionID: {}", executionID); - } - - String workflowPath = String.valueOf(workflowBusiness.getRawApplicationDescriptorPath(currentUser, executionName, version)); - logger.info(workflowPath); - File workflowFile = new File(workflowPath); - if (workflowFile.exists()) { - Files.copy(workflowFile.toPath(), new File(reproVipDir, workflowFile.getName()).toPath(), StandardCopyOption.REPLACE_EXISTING); - logger.info("Workflow file successfully copied to ReproVip directory"); - } else { - logger.warn("Workflow file not found"); + if ( ! Files.exists(reproVipDir)) { + Files.createDirectories(reproVipDir); } + } catch (IOException e) { + logger.error("Exception creating the a reprovip directory {}", reproVipDir, e); + throw new RuntimeException(e); + } + copyProvenanceFiles(reproVipDir, executionID); + return generateReprovipJson(reproVipDir, executionName, executionID, version, currentUser); + } - logger.info("Workflows path: " + server.getWorkflowsPath()); + public void copyProvenanceFiles(Path reproVipDir, String executionID) { + try { + logger.debug("Workflows path: " + server.getWorkflowsPath()); - String provenanceDirPath = server.getWorkflowsPath() + "/" + executionID + "/provenance"; - File provenanceDir = new File(provenanceDirPath); - if (!provenanceDir.exists()) { - logger.warn("Provenance directory does not exist: " + provenanceDirPath); + Path provenanceDirPath = Paths.get(server.getWorkflowsPath() + "/" + executionID + "/provenance"); + if ( ! Files.exists(provenanceDirPath)) { + logger.error("Provenance directory does not exist: " + provenanceDirPath); return; } - File[] allFiles = provenanceDir.listFiles(); - if (allFiles != null) { - for (File file : allFiles) { - logger.info("Found file in provenance directory: " + file.getName()); - } - } - FilenameFilter filter = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.endsWith(".sh.provenance.json"); - } - }; - - File[] matchingFiles = provenanceDir.listFiles(filter); - - if (matchingFiles != null && matchingFiles.length > 0) { - File provenanceFile = matchingFiles[0]; - if (provenanceFile.exists()) { - Files.copy(provenanceFile.toPath(), new File(reproVipDir, provenanceFile.getName()).toPath(), StandardCopyOption.REPLACE_EXISTING); - logger.info(provenanceFile.getName() + " file successfully copied to ReproVip directory"); - } else { - logger.warn(provenanceFile.getName() + " file not found"); + try (Stream stream = Files.list(provenanceDirPath)) { + List provenanceFiles = stream + .filter(path -> path.toString().endsWith(".sh.provenance.json")) + .collect(Collectors.toList()); + for (Path provenanceFile : provenanceFiles) { + Files.copy(provenanceFile, reproVipDir.resolve(provenanceFile.getFileName()), StandardCopyOption.REPLACE_EXISTING); + logger.info("{} file successfully copied to ReproVip directory", provenanceFile); } - } else { - logger.warn("No matching provenance file found"); } } catch (IOException e) { - logger.error("Exception while copying files to ReproVip directory", e); - throw new RuntimeException(e); - } catch (BusinessException e) { + logger.error("Exception creating the a reprovip directory {}", reproVipDir, e); throw new RuntimeException(e); } } - public List getFilesToCopyPaths(String executionName, String executionID, String version, User currentUser) throws BusinessException { + public List getFilesToCopyPaths(String executionName, String executionID, String version) throws BusinessException { List paths = new ArrayList<>(); - List outputData = workflowBusiness.getOutputData(executionID, currentUser.getFolder(), true); + List outputData = workflowBusiness.getRawOutputData(executionID); if (outputData != null && !outputData.isEmpty()) { String outputPath = outputData.get(0).getPath(); if (outputPath != null) { @@ -257,24 +215,9 @@ public List getFilesToCopyPaths(String executionName, String executionID } } - String workflowPath = String.valueOf(workflowBusiness.getRawApplicationDescriptorPath(currentUser, executionName, version)); - if (workflowPath != null && !workflowPath.isEmpty()) { - paths.add(workflowPath); - } - - String provenanceDirPath = server.getWorkflowsPath() + "/" + executionID + "/provenance"; - File provenanceDir = new File(provenanceDirPath); - if (provenanceDir.exists()) { - FilenameFilter filter = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.endsWith(".sh.provenance.json"); - } - }; - File[] matchingFiles = provenanceDir.listFiles(filter); - if (matchingFiles != null && matchingFiles.length > 0) { - paths.add(matchingFiles[0].getAbsolutePath()); - } + AppVersion appVersion = applicationBusiness.getVersion(executionName, version); + if (appVersion != null && appVersion.getJsonLfn() != null) { + paths.add(appVersion.getJsonLfn()); } return paths; diff --git a/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/business/WorkflowBusiness.java b/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/business/WorkflowBusiness.java index ff8404973..f8147f436 100644 --- a/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/business/WorkflowBusiness.java +++ b/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/business/WorkflowBusiness.java @@ -564,7 +564,18 @@ public Simulation getSimulation(String simulationID, boolean refresh) return simulation; } + public List getOutputData( + String simulationID, String currentUserFolder) throws BusinessException { + return getOutputData(simulationID, currentUserFolder, false); + } + + public List getRawOutputData( + String simulationID) throws BusinessException { + return getOutputData(simulationID, null, true); + } + + private List getOutputData( String simulationID, String currentUserFolder, boolean useRawPath) throws BusinessException { @@ -581,13 +592,9 @@ public List getOutputData( list.add(new InOutData(path, output.getOutputID().getProcessor(), output.getType().name())); } - } catch (WorkflowsDBDAOException ex) { + } catch (WorkflowsDBDAOException | DataManagerException ex) { logger.error("Error getting output data for {}", simulationID, ex); throw new BusinessException(ex); - } catch (DataManagerException ex) { - if (!useRawPath) { - throw new BusinessException(ex); - } } return list; } diff --git a/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/dao/h2/SimulationData.java b/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/dao/h2/SimulationData.java index 690dc5919..618ecbfb7 100644 --- a/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/dao/h2/SimulationData.java +++ b/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/dao/h2/SimulationData.java @@ -414,6 +414,7 @@ public Map getNodeCountriesMap() throws DAOException { + "GROUP BY country ORDER BY country"); ps.setString(1, TaskStatus.COMPLETED.name()); ResultSet rs = ps.executeQuery(); + Map map = new HashMap(); while (rs.next()) { diff --git a/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/rpc/ReproVipServiceImpl.java b/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/rpc/ReproVipServiceImpl.java index 15400f9be..d5bcf087a 100644 --- a/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/rpc/ReproVipServiceImpl.java +++ b/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/rpc/ReproVipServiceImpl.java @@ -1,7 +1,5 @@ package fr.insalyon.creatis.vip.application.server.rpc; -import com.google.gwt.user.client.rpc.AsyncCallback; -import fr.insalyon.creatis.vip.application.client.view.ApplicationException; import fr.insalyon.creatis.vip.application.server.business.ReproVipBusiness; import fr.insalyon.creatis.vip.core.client.bean.Execution; import fr.insalyon.creatis.vip.application.client.rpc.ReproVipService; @@ -13,10 +11,6 @@ import fr.insalyon.creatis.vip.core.server.rpc.AbstractRemoteServiceServlet; import javax.servlet.ServletException; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.concurrent.CompletableFuture; public class ReproVipServiceImpl extends AbstractRemoteServiceServlet implements ReproVipService { private ReproVipBusiness reproVipBusiness; @@ -56,28 +50,10 @@ public void updateExecution(String executionID, String newStatus) throws CoreExc throw new CoreException("Failed to update execution", e); } } - @Override - public void executionOutputData(String executionID) throws CoreException { - try { - User currentUser = getSessionUser(); - reproVipBusiness.executionOutputData(executionID, currentUser); - } catch (BusinessException | ApplicationException e) { - throw new RuntimeException(e); - } - } - public String downloadJsonOutputData(String executionName, String executionID, String version) throws CoreException { - try { - User currentUser = getSessionUser(); - String json = reproVipBusiness.createJsonOutputData(executionName, executionID, version, currentUser); - return json; - } catch (BusinessException | ApplicationException e) { - throw new RuntimeException(e); - } - } - public void createReproVipDirectory(String executionName, String executionID, String version) { + public String createReproVipDirectory(String executionName, String executionID, String version) { try { User currentUser = getSessionUser(); - reproVipBusiness.createReproVipDirectory(executionName, executionID, version, currentUser); + return reproVipBusiness.createReproVipDirectory(executionName, executionID, version, currentUser); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/rpc/WorkflowServiceImpl.java b/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/rpc/WorkflowServiceImpl.java index 48a50a5aa..d11b4a43b 100644 --- a/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/rpc/WorkflowServiceImpl.java +++ b/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/rpc/WorkflowServiceImpl.java @@ -686,7 +686,7 @@ public List getPerformanceStats(List simulationList, int typ public List getOutputData(String simulationID) throws ApplicationException { try { return workflowBusiness.getOutputData( - simulationID, getSessionUser().getFolder(), false); + simulationID, getSessionUser().getFolder()); } catch (BusinessException | CoreException ex) { throw new ApplicationException(ex); }