Skip to content

Commit

Permalink
Merge pull request #6031 from BartChris/multi_value_mass_import_backport
Browse files Browse the repository at this point in the history 
[3.6] Backport of - Allow multiple values for same key in mass import csv
  • Loading branch information
@solth
solth authored May 7, 2024
2 parents 6eeaf80 + 192ef04 commit e8a02ae
Show file tree
Hide file tree
Showing 7 changed files with 173 additions and 149 deletions.
6 changes: 3 additions & 3 deletions Kitodo/src/main/java/org/kitodo/production/forms/massimport/MassImportForm.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ public void startMassImport() {
importSuccessMap = new HashMap<>();
PrimeFaces.current().ajax().update("massImportResultDialog");
try {
Map<String, Map<String, String>> presetMetadata = massImportService.prepareMetadata(metadataKeys, records);
Map<String, Map<String, List<String>>> presetMetadata = massImportService.prepareMetadata(metadataKeys, records);
importRecords(presetMetadata);
PrimeFaces.current().executeScript("PF('massImportResultDialog').show();");
PrimeFaces.current().ajax().update("massImportResultDialog");
Expand All @@ -174,10 +174,10 @@ public void prepare() {
*
* @param processMetadata Map containing record IDs as keys and preset metadata lists as values
*/
private void importRecords(Map<String, Map<String, String>> processMetadata) {
private void importRecords(Map<String, Map<String, List<String>>> processMetadata) {
ImportService importService = ServiceManager.getImportService();
PrimeFaces.current().ajax().update("massImportProgressDialog");
for (Map.Entry<String, Map<String, String>> entry : processMetadata.entrySet()) {
for (Map.Entry<String, Map<String, List<String>>> entry : processMetadata.entrySet()) {
try {
importService.importProcess(entry.getKey(), projectId, templateId, importConfiguration,
entry.getValue());
Expand Down
18 changes: 10 additions & 8 deletions Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1173,7 +1173,7 @@ public static void processTempProcess(TempProcess tempProcess, RulesetManagement
* @return the importedProcess
*/
public Process importProcess(String ppn, int projectId, int templateId, ImportConfiguration importConfiguration,
Map<String, String> presetMetadata) throws ImportException {
Map<String, List<String>> presetMetadata) throws ImportException {
LinkedList<TempProcess> processList = new LinkedList<>();
TempProcess tempProcess;
Template template;
Expand Down Expand Up @@ -1256,14 +1256,16 @@ private static Collection<String> getFunctionalMetadata(Ruleset ruleset, Functio
return rulesetManagement.getFunctionalKeys(metadata);
}

private List<MetadataEntry> createMetadata(Map<String, String> presetMetadata) {
private List<MetadataEntry> createMetadata(Map<String, List<String>> presetMetadata) {
List<MetadataEntry> metadata = new LinkedList<>();
for (Map.Entry<String, String> presetMetadataEntry : presetMetadata.entrySet()) {
MetadataEntry metadataEntry = new MetadataEntry();
metadataEntry.setKey(presetMetadataEntry.getKey());
metadataEntry.setValue(presetMetadataEntry.getValue());
metadataEntry.setDomain(MdSec.DMD_SEC);
metadata.add(metadataEntry);
for (Map.Entry<String, List<String>> presetMetadataEntry : presetMetadata.entrySet()) {
for (String presetMetadataEntryValue : presetMetadataEntry.getValue()) {
MetadataEntry metadataEntry = new MetadataEntry();
metadataEntry.setKey(presetMetadataEntry.getKey());
metadataEntry.setValue(presetMetadataEntryValue);
metadataEntry.setDomain(MdSec.DMD_SEC);
metadata.add(metadataEntry);
}
}
return metadata;
}
Expand Down
9 changes: 5 additions & 4 deletions Kitodo/src/main/java/org/kitodo/production/services/data/MassImportService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,16 +114,17 @@ public List<CsvRecord> parseLines(List<String> lines, String separator) {
* @param metadataKeys metadata keys for additional metadata added to individual records during import
* @param records list of CSV records
*/
public Map<String, Map<String, String>> prepareMetadata(List<String> metadataKeys, List<CsvRecord> records)
public Map<String, Map<String, List<String>>> prepareMetadata(List<String> metadataKeys, List<CsvRecord> records)
throws ImportException {
Map<String, Map<String, String>> presetMetadata = new LinkedHashMap<>();
Map<String, Map<String, List<String>>> presetMetadata = new LinkedHashMap<>();
for (CsvRecord record : records) {
Map<String, String> processMetadata = new HashMap<>();
Map<String, List<String>> processMetadata = new HashMap<>();
// skip first metadata key as it always contains the record ID to be used for search
for (int index = 1; index < metadataKeys.size(); index++) {
String metadataKey = metadataKeys.get(index);
if (StringUtils.isNotBlank(metadataKey)) {
processMetadata.put(metadataKey, record.getCsvCells().get(index).getValue());
List<String> values = processMetadata.computeIfAbsent(metadataKey, k -> new ArrayList<>());
values.add(record.getCsvCells().get(index).getValue());
}
}
presetMetadata.put(record.getCsvCells().get(0).getValue(), processMetadata);
Expand Down
67 changes: 67 additions & 0 deletions Kitodo/src/test/java/org/kitodo/production/services/data/ImportServiceIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,11 @@
import java.nio.file.Paths;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.SystemUtils;
import org.junit.AfterClass;
Expand All @@ -39,13 +43,17 @@
import org.kitodo.ExecutionPermission;
import org.kitodo.MockDatabase;
import org.kitodo.SecurityTestUtils;
import org.kitodo.api.Metadata;
import org.kitodo.api.MetadataEntry;
import org.kitodo.api.dataformat.Workpiece;
import org.kitodo.config.ConfigCore;
import org.kitodo.config.enums.ParameterCore;
import org.kitodo.data.database.beans.ImportConfiguration;
import org.kitodo.data.database.beans.Process;
import org.kitodo.data.database.beans.UrlParameter;
import org.kitodo.data.database.beans.User;
import org.kitodo.data.database.exceptions.DAOException;
import org.kitodo.data.exceptions.DataException;
import org.kitodo.exceptions.ImportException;
import org.kitodo.production.services.ServiceManager;

Expand All @@ -59,8 +67,12 @@ public class ImportServiceIT {
private static final String TEST_FILE_ERROR_RESPONSE_PATH = "src/test/resources/customInterfaceErrorResponse.xml";
private static final String RECORD_ID = "11111";
private static final String CUSTOM_INTERFACE_RECORD_ID = "12345";
private static final String TITLE = "Title";
private static final String PLACE = "Place";
private static final int PORT = 8888;
private static final String firstProcess = "First process";
private static final File ORIGINAL_META_10 = new File("src/test/resources/metadata/10/meta.xml");
private static final File BACKUP_META_10 = new File("src/test/resources/metadata/10/meta.xml.1");

@BeforeClass
public static void prepareDatabase() throws Exception {
Expand All @@ -79,13 +91,17 @@ public static void prepareDatabase() throws Exception {
});
server = new StubServer(PORT).run();
setupServer();
FileUtils.copyFile(ORIGINAL_META_10, BACKUP_META_10);
}


@AfterClass
public static void cleanDatabase() throws Exception {
MockDatabase.stopNode();
MockDatabase.cleanDatabase();
server.stop();
FileUtils.deleteQuietly(ORIGINAL_META_10);
FileUtils.moveFile(BACKUP_META_10, ORIGINAL_META_10);
}

@Test
Expand All @@ -98,6 +114,42 @@ public void testImportProcess() throws Exception {
Assert.assertEquals("Not the correct amount of processes found", 8, (long) processService.count());
}

/**
* Tests whether basic catalog metadata import with additional preset metadata to a single process succeeds or not.
*
* @throws DAOException when loading ImportConfiguration or removing test process from test database fails.
* @throws ImportException when importing metadata fails
* @throws IOException when importing metadata fails
*/
@Test
public void testImportProcessWithAdditionalMetadata() throws DAOException, ImportException, IOException, DataException {
Map<String, List<String>> presetMetadata = new HashMap<>();
presetMetadata.put(TITLE, List.of("Band 1"));
presetMetadata.put(PLACE, List.of("Hamburg", "Berlin"));
Process processWithAdditionalMetadata = importProcessWithAdditionalMetadata(RECORD_ID,
MockDatabase.getK10PlusImportConfiguration(), presetMetadata);
Workpiece workpiece = ServiceManager.getMetsService()
.loadWorkpiece(processService.getMetadataFileUri(processWithAdditionalMetadata));
HashSet<Metadata> metadata = workpiece.getLogicalStructure().getMetadata();
try {
Assert.assertTrue("Process does not contain correct metadata",
assertMetadataSetContainsMetadata(metadata, TITLE, "Band 1"));
Assert.assertTrue("Process does not contain correct metadata",
assertMetadataSetContainsMetadata(metadata, PLACE, "Hamburg"));
Assert.assertTrue("Process does not contain correct metadata",
assertMetadataSetContainsMetadata(metadata, PLACE, "Berlin"));
} finally {
ProcessService.deleteProcess(processWithAdditionalMetadata.getId());
}
}

private boolean assertMetadataSetContainsMetadata(HashSet<Metadata> metadataSet, String metadataKey, String metadataValue) {
return metadataSet.stream()
.filter(metadata -> metadata.getKey().equals(metadataKey))
.anyMatch(metadata -> metadata instanceof MetadataEntry &&
((MetadataEntry) metadata).getValue().equals(metadataValue));
}

@Test
public void shouldCreateUrlWithCustomParameters() throws DAOException, ImportException, IOException {
Process importedProcess = importProcess(CUSTOM_INTERFACE_RECORD_ID, MockDatabase.getCustomTypeImportConfiguration());
Expand Down Expand Up @@ -160,4 +212,19 @@ private Process importProcess(String recordId, ImportConfiguration importConfigu
}
return importedProcess;
}

private Process importProcessWithAdditionalMetadata(String recordId, ImportConfiguration importConfiguration,
Map<String, List<String>> presetMetadata)
throws IOException, ImportException {
File script = new File(ConfigCore.getParameter(ParameterCore.SCRIPT_CREATE_DIR_META));
if (!SystemUtils.IS_OS_WINDOWS) {
ExecutionPermission.setExecutePermission(script);
}
Process importedProcess = importService.importProcess(recordId, 1, 1,
importConfiguration, presetMetadata);
if (!SystemUtils.IS_OS_WINDOWS) {
ExecutionPermission.setNoExecutePermission(script);
}
return importedProcess;
}
}
Loading

0 comments on commit e8a02ae

Please sign in to comment.