Skip to content

Commit

Permalink
Merge pull request #5998 from BartChris/mass_import_multi_value
Browse files Browse the repository at this point in the history 
Allow multiple values for same key in mass import csv
  • Loading branch information
@solth
solth authored Apr 10, 2024
2 parents f73e4fe + 7af4367 commit 5e5a86e
Show file tree
Hide file tree
Showing 5 changed files with 105 additions and 19 deletions.
6 changes: 3 additions & 3 deletions Kitodo/src/main/java/org/kitodo/production/forms/massimport/MassImportForm.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ public void startMassImport() {
importSuccessMap = new HashMap<>();
PrimeFaces.current().ajax().update("massImportResultDialog");
try {
Map<String, Map<String, String>> presetMetadata = massImportService.prepareMetadata(metadataKeys, records);
Map<String, Map<String, List<String>>> presetMetadata = massImportService.prepareMetadata(metadataKeys, records);
importRecords(presetMetadata);
PrimeFaces.current().executeScript("PF('massImportResultDialog').show();");
PrimeFaces.current().ajax().update("massImportResultDialog");
Expand Down Expand Up @@ -181,10 +181,10 @@ public void prepare() {
*
* @param processMetadata Map containing record IDs as keys and preset metadata lists as values
*/
private void importRecords(Map<String, Map<String, String>> processMetadata) {
private void importRecords(Map<String, Map<String, List<String>>> processMetadata) {
ImportService importService = ServiceManager.getImportService();
PrimeFaces.current().ajax().update("massImportProgressDialog");
for (Map.Entry<String, Map<String, String>> entry : processMetadata.entrySet()) {
for (Map.Entry<String, Map<String, List<String>>> entry : processMetadata.entrySet()) {
try {
importService.importProcess(entry.getKey(), projectId, templateId, importConfiguration,
entry.getValue());
Expand Down
18 changes: 10 additions & 8 deletions Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1174,7 +1174,7 @@ public static void processTempProcess(TempProcess tempProcess, RulesetManagement
* @return the importedProcess
*/
public Process importProcess(String ppn, int projectId, int templateId, ImportConfiguration importConfiguration,
Map<String, String> presetMetadata) throws ImportException {
Map<String, List<String>> presetMetadata) throws ImportException {
LinkedList<TempProcess> processList = new LinkedList<>();
TempProcess tempProcess;
Template template;
Expand Down Expand Up @@ -1257,14 +1257,16 @@ private static Collection<String> getFunctionalMetadata(Ruleset ruleset, Functio
return rulesetManagement.getFunctionalKeys(metadata);
}

private List<MetadataEntry> createMetadata(Map<String, String> presetMetadata) {
private List<MetadataEntry> createMetadata(Map<String, List<String>> presetMetadata) {
List<MetadataEntry> metadata = new LinkedList<>();
for (Map.Entry<String, String> presetMetadataEntry : presetMetadata.entrySet()) {
MetadataEntry metadataEntry = new MetadataEntry();
metadataEntry.setKey(presetMetadataEntry.getKey());
metadataEntry.setValue(presetMetadataEntry.getValue());
metadataEntry.setDomain(MdSec.DMD_SEC);
metadata.add(metadataEntry);
for (Map.Entry<String, List<String>> presetMetadataEntry : presetMetadata.entrySet()) {
for (String presetMetadataEntryValue : presetMetadataEntry.getValue()) {
MetadataEntry metadataEntry = new MetadataEntry();
metadataEntry.setKey(presetMetadataEntry.getKey());
metadataEntry.setValue(presetMetadataEntryValue);
metadataEntry.setDomain(MdSec.DMD_SEC);
metadata.add(metadataEntry);
}
}
return metadata;
}
Expand Down
9 changes: 5 additions & 4 deletions Kitodo/src/main/java/org/kitodo/production/services/data/MassImportService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,16 +114,17 @@ public List<CsvRecord> parseLines(List<String> lines, String separator) {
* @param metadataKeys metadata keys for additional metadata added to individual records during import
* @param records list of CSV records
*/
public Map<String, Map<String, String>> prepareMetadata(List<String> metadataKeys, List<CsvRecord> records)
public Map<String, Map<String, List<String>>> prepareMetadata(List<String> metadataKeys, List<CsvRecord> records)
throws ImportException {
Map<String, Map<String, String>> presetMetadata = new LinkedHashMap<>();
Map<String, Map<String, List<String>>> presetMetadata = new LinkedHashMap<>();
for (CsvRecord record : records) {
Map<String, String> processMetadata = new HashMap<>();
Map<String, List<String>> processMetadata = new HashMap<>();
// skip first metadata key as it always contains the record ID to be used for search
for (int index = 1; index < metadataKeys.size(); index++) {
String metadataKey = metadataKeys.get(index);
if (StringUtils.isNotBlank(metadataKey)) {
processMetadata.put(metadataKey, record.getCsvCells().get(index).getValue());
List<String> values = processMetadata.computeIfAbsent(metadataKey, k -> new ArrayList<>());
values.add(record.getCsvCells().get(index).getValue());
}
}
presetMetadata.put(record.getCsvCells().get(0).getValue(), processMetadata);
Expand Down
36 changes: 32 additions & 4 deletions Kitodo/src/test/java/org/kitodo/production/services/catalogimport/MassImportTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,11 @@ public class MassImportTest {
private static final String CSV_SECOND_LINE = "456, Band 2, Dresden";
private static final String CSV_THIRD_LINE = "789, Band 3, Berlin";
private static final List<String> CSV_LINES = Arrays.asList(CSV_FIRST_LINE, CSV_SECOND_LINE, CSV_THIRD_LINE);
private static final List<String> METADATA_KEYS_MUTLIPLE_VALUES = Arrays.asList(ID, TITLE, PLACE, PLACE);
private static final String CSV_FIRST_LINE_MUTLIPLE_VALUES = "321, Band 1, Hamburg, Berlin";
private static final String CSV_SECOND_LINE_MUTLIPLE_VALUES = "654, Band 2, Dresden, Hannover";
private static final List<String> CSV_LINES_MUTLIPLE_VALUES = Arrays.asList(CSV_FIRST_LINE_MUTLIPLE_VALUES,
CSV_SECOND_LINE_MUTLIPLE_VALUES);

/**
* Tests parsing CSV lines into CSV records with multiple cells.
Expand Down Expand Up @@ -93,11 +98,34 @@ public void shouldUpdateSeparator() {
public void shouldPrepareMetadata() throws ImportException {
MassImportService service = ServiceManager.getMassImportService();
List<CsvRecord> csvRecords = service.parseLines(CSV_LINES, StringConstants.COMMA_DELIMITER);
Map<String, Map<String, String>> metadata = service.prepareMetadata(METADATA_KEYS, csvRecords);
Map<String, Map<String, List<String>>> metadata = service.prepareMetadata(METADATA_KEYS, csvRecords);
Assert.assertEquals("Wrong number of metadata sets prepared", 3, metadata.size());
Map<String, String> metadataSet = metadata.get("123");
Map<String, List<String>> metadataSet = metadata.get("123");
Assert.assertNotNull("Metadata for record with ID 123 is null", metadataSet);
Assert.assertTrue("Metadata for record with ID 123 does not contain title metadata", metadataSet.containsKey(TITLE));
Assert.assertEquals("Metadata for record with ID 123 contains wrong title", "Band 1", metadataSet.get(TITLE));
Assert.assertEquals("Wrong number of metadata sets prepared", 2,
metadataSet.size());
Assert.assertEquals("Metadata for record with ID 123 contains wrong title", "Band 1",
metadataSet.get(TITLE).get(0));
Assert.assertEquals("Metadata for record with ID 123 has wrong size of place list",
1, metadataSet.get(PLACE).size());
Assert.assertEquals("Metadata for record with ID 123 contains wrong place", "Hamburg",
metadataSet.get(PLACE).get(0));

List<CsvRecord> csvRecordsMultipleValues = service.parseLines(CSV_LINES_MUTLIPLE_VALUES,
StringConstants.COMMA_DELIMITER);
Map<String, Map<String, List<String>>> metadataMultipleValues = service.
prepareMetadata(METADATA_KEYS_MUTLIPLE_VALUES, csvRecordsMultipleValues);
Map<String, List<String>> metadataSetMultipleValues = metadataMultipleValues.get("321");
Assert.assertNotNull("Metadata for record with ID 321 is null", metadataSetMultipleValues);
Assert.assertEquals("Metadata for record with ID 321 contains wrong title", 2,
metadataSetMultipleValues.size());
Assert.assertTrue("Metadata for record with ID 321 does not contain place metadata",
metadataSetMultipleValues.containsKey(PLACE));
Assert.assertEquals("Metadata for record with ID 123 has wrong size of place list", 2,
metadataSetMultipleValues.get(PLACE).size());
Assert.assertEquals("Metadata for record with ID 321 contains wrong place", "Hamburg",
metadataSetMultipleValues.get(PLACE).get(0));
Assert.assertEquals("Metadata for record with ID 321 contains wrong place", "Berlin",
metadataSetMultipleValues.get(PLACE).get(1));
}
}
55 changes: 55 additions & 0 deletions Kitodo/src/test/java/org/kitodo/production/services/data/ImportServiceIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,10 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;

Expand Down Expand Up @@ -108,6 +110,8 @@ public class ImportServiceIT {
private static final int TEMPLATE_ID = 1;
private static final int PROJECT_ID = 1;
private static final int RULESET_ID = 1;
private static final String TITLE = "Title";
private static final String PLACE = "Place";
private static final int EXPECTED_NR_OF_CHILDREN = 23;
private static final String PICA_XML = "picaxml";
private static final String PICA_PPN = "pica.ppn";
Expand Down Expand Up @@ -167,6 +171,42 @@ public void testImportProcess() throws DAOException, DataException, ImportExcept
}
}

/**
* Tests whether basic catalog metadata import with additional preset metadata to a single process succeeds or not.
*
* @throws DAOException when loading ImportConfiguration or removing test process from test database fails.
* @throws ImportException when importing metadata fails
* @throws IOException when importing metadata fails
*/
@Test
public void testImportProcessWithAdditionalMetadata() throws DAOException, ImportException, IOException {
Map<String, List<String>> presetMetadata = new HashMap<>();
presetMetadata.put(TITLE, List.of("Band 1"));
presetMetadata.put(PLACE, List.of("Hamburg", "Berlin"));
Process processWithAdditionalMetadata = importProcessWithAdditionalMetadata(RECORD_ID,
MockDatabase.getK10PlusImportConfiguration(), presetMetadata);
Workpiece workpiece = ServiceManager.getMetsService()
.loadWorkpiece(processService.getMetadataFileUri(processWithAdditionalMetadata));
HashSet<Metadata> metadata = workpiece.getLogicalStructure().getMetadata();
try {
Assert.assertTrue("Process does not contain correct metadata",
assertMetadataSetContainsMetadata(metadata, TITLE, "Band 1"));
Assert.assertTrue("Process does not contain correct metadata",
assertMetadataSetContainsMetadata(metadata, PLACE, "Hamburg"));
Assert.assertTrue("Process does not contain correct metadata",
assertMetadataSetContainsMetadata(metadata, PLACE, "Berlin"));
} finally {
ProcessTestUtils.removeTestProcess(processWithAdditionalMetadata.getId());
}
}

private boolean assertMetadataSetContainsMetadata(HashSet<Metadata> metadataSet, String metadataKey, String metadataValue) {
return metadataSet.stream()
.filter(metadata -> metadata.getKey().equals(metadataKey))
.anyMatch(metadata -> metadata instanceof MetadataEntry &&
((MetadataEntry) metadata).getValue().equals(metadataValue));
}

@Test
public void shouldCreateUrlWithCustomParameters() throws DAOException, ImportException, IOException {
Process importedProcess = importProcess(CUSTOM_INTERFACE_RECORD_ID, MockDatabase.getCustomTypeImportConfiguration());
Expand Down Expand Up @@ -549,4 +589,19 @@ private Process importProcess(String recordId, ImportConfiguration importConfigu
}
return importedProcess;
}

private Process importProcessWithAdditionalMetadata(String recordId, ImportConfiguration importConfiguration,
Map<String, List<String>> presetMetadata)
throws IOException, ImportException {
File script = new File(ConfigCore.getParameter(ParameterCore.SCRIPT_CREATE_DIR_META));
if (!SystemUtils.IS_OS_WINDOWS) {
ExecutionPermission.setExecutePermission(script);
}
Process importedProcess = importService.importProcess(recordId, 1, 1,
importConfiguration, presetMetadata);
if (!SystemUtils.IS_OS_WINDOWS) {
ExecutionPermission.setNoExecutePermission(script);
}
return importedProcess;
}
}

0 comments on commit 5e5a86e

Please sign in to comment.