From 82f03f5f64ea72c1fc5496dfa7d43a57ae965b22 Mon Sep 17 00:00:00 2001 From: Alexandre Cornier Date: Thu, 2 Nov 2023 13:49:35 +0100 Subject: [PATCH] Copy provenance files to subfolders --- .../application/MakeExecutionPublicTab.java | 4 +- .../server/business/ReproVipBusiness.java | 46 +++++++++++++++---- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/vip-application/src/main/java/fr/insalyon/creatis/vip/application/client/view/system/application/MakeExecutionPublicTab.java b/vip-application/src/main/java/fr/insalyon/creatis/vip/application/client/view/system/application/MakeExecutionPublicTab.java index 8231574ac..9cd5db74b 100644 --- a/vip-application/src/main/java/fr/insalyon/creatis/vip/application/client/view/system/application/MakeExecutionPublicTab.java +++ b/vip-application/src/main/java/fr/insalyon/creatis/vip/application/client/view/system/application/MakeExecutionPublicTab.java @@ -1,9 +1,7 @@ package fr.insalyon.creatis.vip.application.client.view.system.application; -import com.google.gwt.json.client.JSONArray; -import com.google.gwt.json.client.JSONObject; -import com.google.gwt.json.client.JSONParser; import com.google.gwt.user.client.rpc.AsyncCallback; +import com.google.gwt.user.client.ui.PopupPanel; import com.smartgwt.client.types.Alignment; import com.smartgwt.client.types.Overflow; import com.smartgwt.client.util.SC; diff --git a/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/business/ReproVipBusiness.java b/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/business/ReproVipBusiness.java index 37e47a8ca..325fe3c2c 100644 --- a/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/business/ReproVipBusiness.java +++ b/vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/business/ReproVipBusiness.java @@ -1,5 +1,6 @@ package fr.insalyon.creatis.vip.application.server.business; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import fr.insalyon.creatis.vip.application.client.bean.AppVersion; import fr.insalyon.creatis.vip.application.client.bean.InOutData; @@ -174,25 +175,50 @@ public List copyProvenanceFiles(Path reproVipDir, String executionID) { logger.debug("Workflows path: " + server.getWorkflowsPath()); List copiedProvenanceFiles = new ArrayList<>(); + // Define the directory where provenance files are stored Path provenanceDirPath = Paths.get(server.getWorkflowsPath() + "/" + executionID + "/provenance"); - if ( ! Files.exists(provenanceDirPath)) { + if (!Files.exists(provenanceDirPath)) { logger.error("Provenance directory does not exist: " + provenanceDirPath); return copiedProvenanceFiles; } + ObjectMapper objectMapper = new ObjectMapper(); try (Stream stream = Files.list(provenanceDirPath)) { List provenanceFiles = stream .filter(path -> path.toString().endsWith(".sh.provenance.json")) .collect(Collectors.toList()); + for (Path provenanceFile : provenanceFiles) { - Files.copy(provenanceFile, reproVipDir.resolve(provenanceFile.getFileName()), StandardCopyOption.REPLACE_EXISTING); - logger.info("{} file successfully copied to ReproVip directory", provenanceFile); - copiedProvenanceFiles.add(reproVipDir.resolve(provenanceFile.getFileName())); + // Read the JSON content of the source file + String jsonContent = Files.readString(provenanceFile); + Map provenanceMap = objectMapper.readValue(jsonContent, new TypeReference>(){}); + // Navigate through the nested JSON to find the md5sum + Map outputFilesSection = (Map) ((Map) provenanceMap.get("public-output")).get("output-files"); + + if (outputFilesSection != null) { + for (Map.Entry entry : outputFilesSection.entrySet()) { + Map fileDetails = (Map) entry.getValue(); + String md5sum = fileDetails.get("md5sum"); + + // Create subfolder named after md5sum + Path md5Dir = reproVipDir.resolve(md5sum); + if (!Files.exists(md5Dir)) { + Files.createDirectories(md5Dir); + } + + // Copy the source file to the new subfolder + Path newLocation = md5Dir.resolve(provenanceFile.getFileName()); + Files.copy(provenanceFile, newLocation, StandardCopyOption.REPLACE_EXISTING); + logger.info("Copied provenance file to directory: {}", newLocation); + + copiedProvenanceFiles.add(newLocation); + } + } } } return copiedProvenanceFiles; } catch (IOException e) { - logger.error("Exception creating the a reprovip directory {}", reproVipDir, e); + logger.error("Error while copying provenance files", e); throw new RuntimeException(e); } } @@ -202,14 +228,16 @@ public List getFilesToCopyPaths(String executionName, String executionID List outputData = workflowBusiness.getRawOutputData(executionID); if (outputData != null && !outputData.isEmpty()) { - String outputPath = outputData.get(0).getPath(); - if (outputPath != null) { - paths.add(outputPath); + for (InOutData data : outputData) { + String outputPath = data.getPath(); + if (outputPath != null) { + paths.add(outputPath); + } } } AppVersion appVersion = applicationBusiness.getVersion(executionName, version); - if (appVersion != null && appVersion.getJsonLfn() != null) { + if (appVersion != null && appVersion.getJsonLfn() != null && !paths.contains(appVersion.getJsonLfn())) { paths.add(appVersion.getJsonLfn()); }