Multi-version import working

src/main/java/nl/knaw/dans/dvingest/DansDepositSupportFactoryImpl.java

@@ -33,12 +33,9 @@ public class DansDepositSupportFactoryImpl implements DansDepositSupportFactory
     private final DansBagMappingService dansBagMappingService;
     private final DataverseService dataverseService;
     private final YamlService yamlService;
-    private final boolean isMigration;
-
-    // todo: property isMigration
 
     @Override
     public Deposit addDansDepositSupportIfEnabled(DataverseIngestDeposit deposit) {
-        return new DansDepositSupport(deposit, isMigration, validateDansBagService, dansBagMappingService, dataverseService, yamlService);
+        return new DansDepositSupport(deposit, validateDansBagService, dansBagMappingService, dataverseService, yamlService);
     }
 }

src/main/java/nl/knaw/dans/dvingest/DdDataverseIngestApplication.java

@@ -92,7 +92,7 @@ public void run(final DdDataverseIngestConfiguration configuration, final Enviro
         if (dansDepositConversionConfig != null) {
             var dansBagMappingServiceImport = createDansBagMappingService(false, dansDepositConversionConfig, dataverseService);
             var validateDansBagImport = new ValidateDansBagServiceImpl(dansDepositConversionConfig.getValidateDansBag(), false);
-            dansDepositSupportFactoryImport = new DansDepositSupportFactoryImpl(validateDansBagImport, dansBagMappingServiceImport, dataverseService, yamlService, false);
+            dansDepositSupportFactoryImport = new DansDepositSupportFactoryImpl(validateDansBagImport, dansBagMappingServiceImport, dataverseService, yamlService);
         }
         var depositTaskFactoryImport = new DepositTaskFactoryImpl(bagProcessorFactory, dansDepositSupportFactoryImport);
         var importJobFactory = new ImportJobFactoryImpl(dataverseIngestDepositFactory, depositTaskFactoryImport);
@@ -107,7 +107,7 @@ public void run(final DdDataverseIngestConfiguration configuration, final Enviro
         if (dansDepositConversionConfig != null) {
             var dansBagMappingServiceMigration = createDansBagMappingService(true, dansDepositConversionConfig, dataverseService);
             var validateDansBagMigration = new ValidateDansBagServiceImpl(dansDepositConversionConfig.getValidateDansBag(), true);
-            dansDepositSupportFactoryMigration = new DansDepositSupportFactoryImpl(validateDansBagMigration, dansBagMappingServiceMigration, dataverseService, yamlService, true);
+            dansDepositSupportFactoryMigration = new DansDepositSupportFactoryImpl(validateDansBagMigration, dansBagMappingServiceMigration, dataverseService, yamlService);
         }
         var depositTaskFactoryMigration = new DepositTaskFactoryImpl(bagProcessorFactory, dansDepositSupportFactoryMigration);
         var migrationJobFactory = new ImportJobFactoryImpl(dataverseIngestDepositFactory, depositTaskFactoryMigration);

src/main/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditor.java

@@ -122,16 +122,34 @@ private void deleteFiles() throws IOException, DataverseException {
         log.debug("End deleting files for deposit {}", depositId);
     }
 
-    private void replaceFiles() throws IOException, DataverseException {
+    private void replaceFiles() throws IOException {
         log.debug("Start replacing {} files for deposit {}", depositId, editFiles.getReplaceFiles().size());
         for (var filepath : editFiles.getReplaceFiles()) {
             log.debug("Replacing file: {}", filepath);
             var fileMeta = filesInDatasetCache.get(filepath);
-            dataverseService.replaceFile(pid, fileMeta, dataDir.resolve(filepath));
+            utilityServices.wrapIfZipFile(dataDir.resolve(filepath)).ifPresentOrElse(
+                zipFile -> {
+                    replaceFileOrThrow(pid, fileMeta, zipFile);
+                    FileUtils.deleteQuietly(zipFile.toFile());
+                },
+                () -> {
+                    var fileToUpload = dataDir.resolve(filepath);
+                    replaceFileOrThrow(pid, fileMeta, fileToUpload);
+                }
+            );
         }
         log.debug("End replacing files for deposit {}", depositId);
     }
 
+    private void replaceFileOrThrow(String pid, FileMeta fileMeta, Path fileToUpload) {
+        try {
+            dataverseService.replaceFile(pid, fileMeta, fileToUpload);
+        }
+        catch (IOException | DataverseException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     private void addRestrictedFiles() throws IOException, DataverseException {
         log.debug("Start adding {} restricted files for deposit {}", editFiles.getAddRestrictedFiles().size(), depositId);
         var iterator = new PathIterator(getRestrictedFilesToUpload());

src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingService.java

@@ -20,6 +20,7 @@
 import nl.knaw.dans.dvingest.core.yaml.EditFiles;
 import nl.knaw.dans.dvingest.core.yaml.EditPermissions;
 import nl.knaw.dans.dvingest.core.yaml.Init;
+import nl.knaw.dans.dvingest.core.yaml.UpdateAction;
 import nl.knaw.dans.lib.dataverse.DataverseException;
 import nl.knaw.dans.lib.dataverse.model.dataset.Dataset;
 import nl.knaw.dans.lib.dataverse.model.dataset.DatasetVersion;
@@ -87,6 +88,14 @@ public interface DansBagMappingService {
      */
     EditPermissions getEditPermissionsFromDansDeposit(DansBagDeposit dansDeposit, String updatesDataset);
 
+    /**
+     * Maps the DANS deposit to an update action for the dataset. This determines how to publish the dataset (as migrated or as new).
+     *
+     * @param dansDeposit the DANS deposit
+     * @return the update action for the dataset
+     */
+    UpdateAction getUpdateActionFromDansDeposit(DansBagDeposit dansDeposit);
+
     /**
      * Packages the original metadata of the DANS bag into a ZIP file and returns the local path to the ZIP file.
      *

src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceImpl.java

@@ -23,6 +23,7 @@
 import nl.knaw.dans.dvingest.core.dansbag.deposit.FileInfo;
 import nl.knaw.dans.dvingest.core.dansbag.exception.InvalidDepositException;
 import nl.knaw.dans.dvingest.core.dansbag.mapper.DepositToDvDatasetMetadataMapper;
+import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.Amd;
 import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.FileElement;
 import nl.knaw.dans.dvingest.core.dansbag.xml.XPathEvaluator;
 import nl.knaw.dans.dvingest.core.dansbag.xml.XmlReader;
@@ -34,10 +35,14 @@
 import nl.knaw.dans.dvingest.core.yaml.Expect;
 import nl.knaw.dans.dvingest.core.yaml.Expect.State;
 import nl.knaw.dans.dvingest.core.yaml.Init;
+import nl.knaw.dans.dvingest.core.yaml.PublishAction;
+import nl.knaw.dans.dvingest.core.yaml.ReleaseMigratedAction;
+import nl.knaw.dans.dvingest.core.yaml.UpdateAction;
 import nl.knaw.dans.lib.dataverse.DataverseException;
 import nl.knaw.dans.lib.dataverse.model.RoleAssignment;
 import nl.knaw.dans.lib.dataverse.model.dataset.Dataset;
 import nl.knaw.dans.lib.dataverse.model.dataset.DatasetVersion;
+import nl.knaw.dans.lib.dataverse.model.dataset.UpdateType;
 import nl.knaw.dans.lib.dataverse.model.user.AuthenticatedUser;
 import nl.knaw.dans.lib.util.ZipUtil;
 import org.apache.commons.lang3.StringUtils;
@@ -130,15 +135,21 @@ public Init getInitFromDansDeposit(DansBagDeposit dansDeposit, boolean isUpdate)
                 throw new IllegalArgumentException("Migration deposit must have a DOI");
             }
             var create = new Create();
-            create.setImportPid(dansDeposit.getDoi());
+            var doi = dansDeposit.getDoi();
+            if (!doi.startsWith("doi:")) {
+                doi = "doi:" + doi;
+            }
+            create.setImportPid(doi);
             var init = new Init();
             init.setCreate(create);
+            return init;
         }
         else if (isUpdate) {
             var expect = new Expect();
             expect.setState(State.released);
             var init = new Init();
             init.setExpect(expect);
+            return init;
         }
         return null;
     }
@@ -200,6 +211,28 @@ public EditPermissions getEditPermissionsFromDansDeposit(DansBagDeposit dansDepo
         }
     }
 
+    @Override
+    public UpdateAction getUpdateActionFromDansDeposit(DansBagDeposit dansDeposit) {
+        if (depositToDvDatasetMetadataMapper.isMigration()) {
+            var amd = dansDeposit.getAmd();
+
+            if (amd == null) {
+                throw new RuntimeException(String.format("no AMD found for %s", dansDeposit.getDoi()));
+            }
+
+            var date = Amd.toPublicationDate(amd);
+
+            if (date.isEmpty()) {
+                throw new IllegalArgumentException(String.format("no publication date found in AMD for %s", dansDeposit.getDoi()));
+            }
+
+            return new ReleaseMigratedAction(date.get());
+        }
+        else {
+            return new PublishAction(UpdateType.major);
+        }
+    }
+
     @Override
     public String packageOriginalMetadata(DansBagDeposit dansDeposit) throws IOException {
         if (!depositToDvDatasetMetadataMapper.isMigration()) {

src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositConverter.java

@@ -16,30 +16,40 @@
 package nl.knaw.dans.dvingest.core.dansbag;
 
 import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import nl.knaw.dans.dvingest.core.dansbag.deposit.DansBagDeposit;
 import nl.knaw.dans.dvingest.core.service.YamlService;
 import nl.knaw.dans.dvingest.core.yaml.EditFilesRoot;
 import nl.knaw.dans.dvingest.core.yaml.EditPermissionsRoot;
 import nl.knaw.dans.dvingest.core.yaml.InitRoot;
 import nl.knaw.dans.dvingest.core.yaml.UpdateStateRoot;
 import nl.knaw.dans.lib.dataverse.model.dataset.DatasetVersion;
+import org.apache.commons.io.FileUtils;
 
 import java.io.IOException;
+import java.util.List;
 
+@Slf4j
 @AllArgsConstructor
 public class DansDepositConverter {
+    private static final List<String> YAML_FILES = List.of("init.yml", "dataset.yml", "edit-files.yml", "edit-permissions.yml", "update-state.yml");
+
     private final DansBagDeposit dansDeposit;
     private final String updatesDataset;
     private final DatasetVersion currentMetadata;
     private final DansBagMappingService mappingService;
     private final YamlService yamlService;
 
     public void run() throws IOException {
+        deleteOldYamlFilesIfPresent();
+
         // TODO: pass to getEditFilesFromDansDeposit so that update-deposit can register it as a replaced file
         var originalMetadataPath = mappingService.packageOriginalMetadata(dansDeposit);
 
         var init = mappingService.getInitFromDansDeposit(dansDeposit, updatesDataset != null);
-        yamlService.writeYaml(new InitRoot(init), dansDeposit.getBagDir().resolve("init.yml"));
+        if (init != null) {
+            yamlService.writeYaml(new InitRoot(init), dansDeposit.getBagDir().resolve("init.yml"));
+        }
 
         var dataset = mappingService.getDatasetMetadataFromDansDeposit(dansDeposit, currentMetadata);
         yamlService.writeYaml(dataset, dansDeposit.getBagDir().resolve("dataset.yml"));
@@ -50,8 +60,20 @@ public void run() throws IOException {
         var editPermissions = mappingService.getEditPermissionsFromDansDeposit(dansDeposit, updatesDataset);
         yamlService.writeYaml(new EditPermissionsRoot(editPermissions), dansDeposit.getBagDir().resolve("edit-permissions.yml"));
 
-        var updateState = new UpdateStateRoot();
-//        updateState.setAction("publish-major");
-        yamlService.writeYaml(updateState, dansDeposit.getBagDir().resolve("update-state.yml"));
+        var updateState = mappingService.getUpdateActionFromDansDeposit(dansDeposit);
+        yamlService.writeYaml(new UpdateStateRoot(updateState), dansDeposit.getBagDir().resolve("update-state.yml"));
+    }
+
+    private void deleteOldYamlFilesIfPresent() {
+        log.debug("Starting with clean slate, deleting old YAML files if present");
+        for (String file : YAML_FILES) {
+            var deleted = FileUtils.deleteQuietly(dansDeposit.getBagDir().resolve(file).toFile());
+            if (deleted) {
+                log.debug("Deleted old YAML file: {}", file);
+            }
+            else {
+                log.debug("No old YAML file found or could not be deleted: {}", file);
+            }
+        }
     }
 }

src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositSupport.java

@@ -28,6 +28,8 @@
 import nl.knaw.dans.dvingest.core.dansbag.exception.RejectedDepositException;
 import nl.knaw.dans.dvingest.core.service.DataverseService;
 import nl.knaw.dans.dvingest.core.service.YamlService;
+import nl.knaw.dans.dvingest.core.yaml.PublishAction;
+import nl.knaw.dans.dvingest.core.yaml.ReleaseMigratedAction;
 import nl.knaw.dans.lib.dataverse.DataverseException;
 import nl.knaw.dans.lib.dataverse.model.dataset.DatasetVersion;
 
@@ -49,14 +51,12 @@ public class DansDepositSupport implements Deposit {
     private final YamlService yamlService;
     private final DataverseIngestDeposit ingestDataverseIngestDeposit;
     private final boolean isDansDeposit;
-    private final boolean isMigration;
 
     private DansBagDeposit dansDeposit;
 
-    public DansDepositSupport(DataverseIngestDeposit dataverseIngestDeposit, boolean isMigration, ValidateDansBagService validateDansBagService, DansBagMappingService dansBagMappingService,
+    public DansDepositSupport(DataverseIngestDeposit dataverseIngestDeposit, ValidateDansBagService validateDansBagService, DansBagMappingService dansBagMappingService,
         DataverseService dataverseService, YamlService yamlService) {
         this.ingestDataverseIngestDeposit = dataverseIngestDeposit;
-        this.isMigration = isMigration;
         this.validateDansBagService = validateDansBagService;
         this.dansBagMappingService = dansBagMappingService;
         this.dataverseService = dataverseService;
@@ -120,26 +120,26 @@ public Path getLocation() {
 
     @Override
     public void onSuccess(@NonNull String pid, String message) {
-//        try {
-//            var bag = ingestDataverseIngestDeposit.getBags().get(0);
-//            var action = bag.getUpdateState().getAction();
-//
-//            if (action.startsWith("publish")) {
-//                handlePublishAction(pid);
-//            }
-//            else if (action.equals("submit-for-review")) {
-//                handleSubmitForReviewAction(pid);
-//            }
-//            else {
-//                throw new RuntimeException("Unknown update action: " + action);
-//            }
-//        }
-//        catch (IOException | ConfigurationException e) {
-//            throw new RuntimeException("Error processing onSuccess", e);
-//        }
+        try {
+            var bag = ingestDataverseIngestDeposit.getBags().get(0);
+            var action = bag.getUpdateState();
+
+            if (action instanceof PublishAction) {
+                handlePublishAction(pid, false);
+            }
+            else if (action instanceof ReleaseMigratedAction) {
+                handlePublishAction(pid, true);
+            }
+            else {
+                throw new RuntimeException("Unknown update action: " + action);
+            }
+        }
+        catch (IOException | ConfigurationException e) {
+            throw new RuntimeException("Error processing onSuccess", e);
+        }
     }
 
-    private void handlePublishAction(String pid) {
+    private void handlePublishAction(String pid, boolean isMigration) {
         try {
             var nbn = dataverseService.getDatasetUrnNbn(pid);
             var newProps = new HashMap<String, String>();
@@ -156,16 +156,6 @@ private void handlePublishAction(String pid) {
         }
     }
 
-    private void handleSubmitForReviewAction(String pid) {
-        var newProps = new HashMap<String, String>();
-        newProps.put("state.label", "SUBMITTED");
-        newProps.put("state.description", "The dataset is submitted for review");
-        if (!isMigration) {
-            newProps.put("identifier.doi", pid);
-        }
-        ingestDataverseIngestDeposit.updateProperties(newProps);
-    }
-
     @Override
     public void onFailed(String pid, String message) {
         ingestDataverseIngestDeposit.onFailed(pid, message);

src/main/java/nl/knaw/dans/dvingest/core/service/DataverseServiceImpl.java

@@ -219,7 +219,7 @@ public String getDatasetState(String pid) throws IOException, DataverseException
     @Override
     public void importDataset(String pid, Dataset dataset) throws IOException, DataverseException {
         log.debug("Start importing dataset for deposit {}", pid);
-        var result = dataverseClient.dataverse("root").importDataset(dataset, pid, false);
+        var result = dataverseClient.dataverse("root").importDataset(dataset, pid, false, metadataKeys);
         log.debug(result.getEnvelopeAsString());
         log.debug("End importing dataset for deposit {}", pid);
     }

src/main/java/nl/knaw/dans/dvingest/core/service/UtilityServices.java

@@ -20,6 +20,7 @@
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.Map;
+import java.util.Optional;
 
 public interface UtilityServices {
 
@@ -28,4 +29,6 @@ public interface UtilityServices {
     PathIteratorZipperBuilder createPathIteratorZipperBuilder();
 
     PathIteratorZipperBuilder createPathIteratorZipperBuilder(Map<String, String> renameMap);
+
+    Optional<Path> wrapIfZipFile(Path path) throws IOException;
 }