From 256e0372e2151a5a16512db8017db7006eefd35e Mon Sep 17 00:00:00 2001 From: BartChris Date: Thu, 7 Mar 2024 18:55:38 +0100 Subject: [PATCH 1/6] add multiple values for same key in mass import csv --- .../forms/massimport/MassImportForm.java | 6 +++--- .../services/data/ImportService.java | 18 ++++++++++-------- .../services/data/MassImportService.java | 9 +++++---- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/Kitodo/src/main/java/org/kitodo/production/forms/massimport/MassImportForm.java b/Kitodo/src/main/java/org/kitodo/production/forms/massimport/MassImportForm.java index d7dbed06d75..87087b4d011 100644 --- a/Kitodo/src/main/java/org/kitodo/production/forms/massimport/MassImportForm.java +++ b/Kitodo/src/main/java/org/kitodo/production/forms/massimport/MassImportForm.java @@ -153,7 +153,7 @@ public void startMassImport() { importSuccessMap = new HashMap<>(); PrimeFaces.current().ajax().update("massImportResultDialog"); try { - Map> presetMetadata = massImportService.prepareMetadata(metadataKeys, records); + Map>> presetMetadata = massImportService.prepareMetadata(metadataKeys, records); importRecords(presetMetadata); PrimeFaces.current().executeScript("PF('massImportResultDialog').show();"); PrimeFaces.current().ajax().update("massImportResultDialog"); @@ -181,10 +181,10 @@ public void prepare() { * * @param processMetadata Map containing record IDs as keys and preset metadata lists as values */ - private void importRecords(Map> processMetadata) { + private void importRecords(Map>> processMetadata) { ImportService importService = ServiceManager.getImportService(); PrimeFaces.current().ajax().update("massImportProgressDialog"); - for (Map.Entry> entry : processMetadata.entrySet()) { + for (Map.Entry>> entry : processMetadata.entrySet()) { try { importService.importProcess(entry.getKey(), projectId, templateId, importConfiguration, entry.getValue()); diff --git a/Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java b/Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java index 48fe88c3470..0bb9a2b95c1 100644 --- a/Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java +++ b/Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java @@ -1174,7 +1174,7 @@ public static void processTempProcess(TempProcess tempProcess, RulesetManagement * @return the importedProcess */ public Process importProcess(String ppn, int projectId, int templateId, ImportConfiguration importConfiguration, - Map presetMetadata) throws ImportException { + Map> presetMetadata) throws ImportException { LinkedList processList = new LinkedList<>(); TempProcess tempProcess; Template template; @@ -1257,14 +1257,16 @@ private static Collection getFunctionalMetadata(Ruleset ruleset, Functio return rulesetManagement.getFunctionalKeys(metadata); } - private List createMetadata(Map presetMetadata) { + private List createMetadata(Map> presetMetadata) { List metadata = new LinkedList<>(); - for (Map.Entry presetMetadataEntry : presetMetadata.entrySet()) { - MetadataEntry metadataEntry = new MetadataEntry(); - metadataEntry.setKey(presetMetadataEntry.getKey()); - metadataEntry.setValue(presetMetadataEntry.getValue()); - metadataEntry.setDomain(MdSec.DMD_SEC); - metadata.add(metadataEntry); + for (Map.Entry> presetMetadataEntry : presetMetadata.entrySet()) { + for (String presetMetadataEntryValue : presetMetadataEntry.getValue()) { + MetadataEntry metadataEntry = new MetadataEntry(); + metadataEntry.setKey(presetMetadataEntry.getKey()); + metadataEntry.setValue(presetMetadataEntryValue); + metadataEntry.setDomain(MdSec.DMD_SEC); + metadata.add(metadataEntry); + } } return metadata; } diff --git a/Kitodo/src/main/java/org/kitodo/production/services/data/MassImportService.java b/Kitodo/src/main/java/org/kitodo/production/services/data/MassImportService.java index 522fed00b9c..fc0c3fd6d0f 100644 --- a/Kitodo/src/main/java/org/kitodo/production/services/data/MassImportService.java +++ b/Kitodo/src/main/java/org/kitodo/production/services/data/MassImportService.java @@ -114,16 +114,17 @@ public List parseLines(List lines, String separator) { * @param metadataKeys metadata keys for additional metadata added to individual records during import * @param records list of CSV records */ - public Map> prepareMetadata(List metadataKeys, List records) + public Map>> prepareMetadata(List metadataKeys, List records) throws ImportException { - Map> presetMetadata = new LinkedHashMap<>(); + Map>> presetMetadata = new LinkedHashMap<>(); for (CsvRecord record : records) { - Map processMetadata = new HashMap<>(); + Map> processMetadata = new HashMap<>(); // skip first metadata key as it always contains the record ID to be used for search for (int index = 1; index < metadataKeys.size(); index++) { String metadataKey = metadataKeys.get(index); if (StringUtils.isNotBlank(metadataKey)) { - processMetadata.put(metadataKey, record.getCsvCells().get(index).getValue()); + List values = processMetadata.computeIfAbsent(metadataKey, k -> new ArrayList<>()); + values.add(record.getCsvCells().get(index).getValue()); } } presetMetadata.put(record.getCsvCells().get(0).getValue(), processMetadata); From 58290406be9906a4aa7219c9da2680c5836774e1 Mon Sep 17 00:00:00 2001 From: BartChris Date: Tue, 2 Apr 2024 14:50:30 +0200 Subject: [PATCH 2/6] extend MassImportTest to cover multiple values for single key --- .../catalogimport/MassImportTest.java | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/Kitodo/src/test/java/org/kitodo/production/services/catalogimport/MassImportTest.java b/Kitodo/src/test/java/org/kitodo/production/services/catalogimport/MassImportTest.java index 062a884099e..ee5d36503c5 100644 --- a/Kitodo/src/test/java/org/kitodo/production/services/catalogimport/MassImportTest.java +++ b/Kitodo/src/test/java/org/kitodo/production/services/catalogimport/MassImportTest.java @@ -30,11 +30,17 @@ public class MassImportTest { private static final String ID = "ID"; private static final String TITLE = "Title"; private static final String PLACE = "Place"; + private static final String SECOND_PLACE = "Place"; private static final List METADATA_KEYS = Arrays.asList(ID, TITLE, PLACE); private static final String CSV_FIRST_LINE = "123, Band 1, Hamburg"; private static final String CSV_SECOND_LINE = "456, Band 2, Dresden"; private static final String CSV_THIRD_LINE = "789, Band 3, Berlin"; private static final List CSV_LINES = Arrays.asList(CSV_FIRST_LINE, CSV_SECOND_LINE, CSV_THIRD_LINE); + private static final List METADATA_KEYS_MUTLIPLE_VALUES = Arrays.asList(ID, TITLE, PLACE, SECOND_PLACE); + private static final String CSV_FIRST_LINE_MUTLIPLE_VALUES = "321, Band 1, Hamburg, Berlin"; + private static final String CSV_SECOND_LINE_MUTLIPLE_VALUES = "654, Band 2, Dresden, Hannover"; + private static final List CSV_LINES_MUTLIPLE_VALUES = Arrays.asList(CSV_FIRST_LINE_MUTLIPLE_VALUES, + CSV_SECOND_LINE_MUTLIPLE_VALUES); /** * Tests parsing CSV lines into CSV records with multiple cells. @@ -93,11 +99,34 @@ public void shouldUpdateSeparator() { public void shouldPrepareMetadata() throws ImportException { MassImportService service = ServiceManager.getMassImportService(); List csvRecords = service.parseLines(CSV_LINES, StringConstants.COMMA_DELIMITER); - Map> metadata = service.prepareMetadata(METADATA_KEYS, csvRecords); + Map>> metadata = service.prepareMetadata(METADATA_KEYS, csvRecords); Assert.assertEquals("Wrong number of metadata sets prepared", 3, metadata.size()); - Map metadataSet = metadata.get("123"); + Map> metadataSet = metadata.get("123"); Assert.assertNotNull("Metadata for record with ID 123 is null", metadataSet); - Assert.assertTrue("Metadata for record with ID 123 does not contain title metadata", metadataSet.containsKey(TITLE)); - Assert.assertEquals("Metadata for record with ID 123 contains wrong title", "Band 1", metadataSet.get(TITLE)); + Assert.assertEquals("Metadata for record with ID 123 contains wrong title", 2, + metadataSet.size()); + Assert.assertEquals("Metadata for record with ID 123 contains wrong title", "Band 1", + metadataSet.get(TITLE).get(0)); + Assert.assertEquals("Metadata for record with ID 123 has wrong size of place list", + 1, metadataSet.get(PLACE).size()); + Assert.assertEquals("Metadata for record with ID 123 contains wrong place", "Hamburg", + metadataSet.get(PLACE).get(0)); + + List csvRecords_multiple_values = service.parseLines(CSV_LINES_MUTLIPLE_VALUES, + StringConstants.COMMA_DELIMITER); + Map>> metadata_multiple_values = service. + prepareMetadata(METADATA_KEYS_MUTLIPLE_VALUES, csvRecords_multiple_values); + Map> metadataSet_multiple_values = metadata_multiple_values.get("321"); + Assert.assertNotNull("Metadata for record with ID 321 is null", metadataSet_multiple_values); + Assert.assertEquals("Metadata for record with ID 321 contains wrong title", 2, + metadataSet_multiple_values.size()); + Assert.assertTrue("Metadata for record with ID 321 does not contain place metadata", + metadataSet_multiple_values.containsKey(PLACE)); + Assert.assertEquals("Metadata for record with ID 123 has wrong size of place list", 2, + metadataSet_multiple_values.get(PLACE).size()); + Assert.assertEquals("Metadata for record with ID 321 has wrong size of place list", "Hamburg", + metadataSet_multiple_values.get(PLACE).get(0)); + Assert.assertEquals("Metadata for record with ID 321 contains wrong place", "Berlin", + metadataSet_multiple_values.get(PLACE).get(1)); } } From 4a147775936e7306af024558bece40614d48efb6 Mon Sep 17 00:00:00 2001 From: BartChris Date: Tue, 2 Apr 2024 15:21:47 +0200 Subject: [PATCH 3/6] rename variables --- .../catalogimport/MassImportTest.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Kitodo/src/test/java/org/kitodo/production/services/catalogimport/MassImportTest.java b/Kitodo/src/test/java/org/kitodo/production/services/catalogimport/MassImportTest.java index ee5d36503c5..1f7a72dba1a 100644 --- a/Kitodo/src/test/java/org/kitodo/production/services/catalogimport/MassImportTest.java +++ b/Kitodo/src/test/java/org/kitodo/production/services/catalogimport/MassImportTest.java @@ -112,21 +112,21 @@ public void shouldPrepareMetadata() throws ImportException { Assert.assertEquals("Metadata for record with ID 123 contains wrong place", "Hamburg", metadataSet.get(PLACE).get(0)); - List csvRecords_multiple_values = service.parseLines(CSV_LINES_MUTLIPLE_VALUES, + List csvRecordsMultipleValues = service.parseLines(CSV_LINES_MUTLIPLE_VALUES, StringConstants.COMMA_DELIMITER); - Map>> metadata_multiple_values = service. - prepareMetadata(METADATA_KEYS_MUTLIPLE_VALUES, csvRecords_multiple_values); - Map> metadataSet_multiple_values = metadata_multiple_values.get("321"); - Assert.assertNotNull("Metadata for record with ID 321 is null", metadataSet_multiple_values); + Map>> metadataMultipleValues = service. + prepareMetadata(METADATA_KEYS_MUTLIPLE_VALUES, csvRecordsMultipleValues); + Map> metadataSetMultipleValues = metadataMultipleValues.get("321"); + Assert.assertNotNull("Metadata for record with ID 321 is null", metadataSetMultipleValues); Assert.assertEquals("Metadata for record with ID 321 contains wrong title", 2, - metadataSet_multiple_values.size()); + metadataSetMultipleValues.size()); Assert.assertTrue("Metadata for record with ID 321 does not contain place metadata", - metadataSet_multiple_values.containsKey(PLACE)); + metadataSetMultipleValues.containsKey(PLACE)); Assert.assertEquals("Metadata for record with ID 123 has wrong size of place list", 2, - metadataSet_multiple_values.get(PLACE).size()); + metadataSetMultipleValues.get(PLACE).size()); Assert.assertEquals("Metadata for record with ID 321 has wrong size of place list", "Hamburg", - metadataSet_multiple_values.get(PLACE).get(0)); + metadataSetMultipleValues.get(PLACE).get(0)); Assert.assertEquals("Metadata for record with ID 321 contains wrong place", "Berlin", - metadataSet_multiple_values.get(PLACE).get(1)); + metadataSetMultipleValues.get(PLACE).get(1)); } } From 8bb66e7056ae4659706f4aa750844654483851e6 Mon Sep 17 00:00:00 2001 From: BartChris Date: Tue, 2 Apr 2024 20:40:07 +0200 Subject: [PATCH 4/6] Add tests for import with additional metadata --- .../services/data/ImportServiceIT.java | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/Kitodo/src/test/java/org/kitodo/production/services/data/ImportServiceIT.java b/Kitodo/src/test/java/org/kitodo/production/services/data/ImportServiceIT.java index 137b2916234..706ea025da1 100644 --- a/Kitodo/src/test/java/org/kitodo/production/services/data/ImportServiceIT.java +++ b/Kitodo/src/test/java/org/kitodo/production/services/data/ImportServiceIT.java @@ -27,8 +27,10 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -108,6 +110,8 @@ public class ImportServiceIT { private static final int TEMPLATE_ID = 1; private static final int PROJECT_ID = 1; private static final int RULESET_ID = 1; + private static final String TITLE = "Title"; + private static final String PLACE = "Place"; private static final int EXPECTED_NR_OF_CHILDREN = 23; private static final String PICA_XML = "picaxml"; private static final String PICA_PPN = "pica.ppn"; @@ -167,6 +171,42 @@ public void testImportProcess() throws DAOException, DataException, ImportExcept } } + /** + * Tests whether basic catalog metadata import with additional preset metadata to a single process succeeds or not. + * + * @throws DAOException when loading ImportConfiguration or removing test process from test database fails. + * @throws ImportException when importing metadata fails + * @throws IOException when importing metadata fails + */ + @Test + public void testImportProcessWithAdditionalMetadata() throws DAOException, ImportException, IOException { + Map> presetMetadata = new HashMap<>(); + presetMetadata.put(TITLE, List.of("Band 1")); + presetMetadata.put(PLACE, List.of("Hamburg", "Berlin")); + Process processWithAdditionalMetadata = importProcessWithAdditionalMetadata(RECORD_ID, + MockDatabase.getK10PlusImportConfiguration(), presetMetadata); + Workpiece workpiece = ServiceManager.getMetsService() + .loadWorkpiece(processService.getMetadataFileUri(processWithAdditionalMetadata)); + HashSet metadata = workpiece.getLogicalStructure().getMetadata(); + try { + Assert.assertTrue("Process does not contain correct metadata", + assertMetadataSetContainsMetadata(metadata,TITLE,"Band 1")); + Assert.assertTrue("Process does not contain correct metadata", + assertMetadataSetContainsMetadata(metadata,PLACE,"Hamburg")); + Assert.assertTrue("Process does not contain correct metadata", + assertMetadataSetContainsMetadata(metadata,PLACE,"Berlin")); + } finally { + ProcessTestUtils.removeTestProcess(processWithAdditionalMetadata.getId()); + } + } + + private boolean assertMetadataSetContainsMetadata(HashSet metadataSet, String metadataKey, String metadataValue) { + return metadataSet.stream() + .filter(metadata -> metadata.getKey().equals(metadataKey)) + .anyMatch(metadata -> metadata instanceof MetadataEntry && + ((MetadataEntry) metadata).getValue().equals(metadataValue)); + } + @Test public void shouldCreateUrlWithCustomParameters() throws DAOException, ImportException, IOException { Process importedProcess = importProcess(CUSTOM_INTERFACE_RECORD_ID, MockDatabase.getCustomTypeImportConfiguration()); @@ -549,4 +589,19 @@ private Process importProcess(String recordId, ImportConfiguration importConfigu } return importedProcess; } + + private Process importProcessWithAdditionalMetadata(String recordId, ImportConfiguration importConfiguration, + Map> presetMetadata) + throws IOException, ImportException { + File script = new File(ConfigCore.getParameter(ParameterCore.SCRIPT_CREATE_DIR_META)); + if (!SystemUtils.IS_OS_WINDOWS) { + ExecutionPermission.setExecutePermission(script); + } + Process importedProcess = importService.importProcess(recordId, 1, 1, + importConfiguration, presetMetadata); + if (!SystemUtils.IS_OS_WINDOWS) { + ExecutionPermission.setNoExecutePermission(script); + } + return importedProcess; + } } From 2a4b8450eb0367e88e7f6856b5d1eabbff721f11 Mon Sep 17 00:00:00 2001 From: BartChris Date: Tue, 9 Apr 2024 19:02:04 +0200 Subject: [PATCH 5/6] implement requested changes from review --- .../production/services/catalogimport/MassImportTest.java | 5 ++--- .../kitodo/production/services/data/ImportServiceIT.java | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Kitodo/src/test/java/org/kitodo/production/services/catalogimport/MassImportTest.java b/Kitodo/src/test/java/org/kitodo/production/services/catalogimport/MassImportTest.java index 1f7a72dba1a..4af4abae117 100644 --- a/Kitodo/src/test/java/org/kitodo/production/services/catalogimport/MassImportTest.java +++ b/Kitodo/src/test/java/org/kitodo/production/services/catalogimport/MassImportTest.java @@ -30,13 +30,12 @@ public class MassImportTest { private static final String ID = "ID"; private static final String TITLE = "Title"; private static final String PLACE = "Place"; - private static final String SECOND_PLACE = "Place"; private static final List METADATA_KEYS = Arrays.asList(ID, TITLE, PLACE); private static final String CSV_FIRST_LINE = "123, Band 1, Hamburg"; private static final String CSV_SECOND_LINE = "456, Band 2, Dresden"; private static final String CSV_THIRD_LINE = "789, Band 3, Berlin"; private static final List CSV_LINES = Arrays.asList(CSV_FIRST_LINE, CSV_SECOND_LINE, CSV_THIRD_LINE); - private static final List METADATA_KEYS_MUTLIPLE_VALUES = Arrays.asList(ID, TITLE, PLACE, SECOND_PLACE); + private static final List METADATA_KEYS_MUTLIPLE_VALUES = Arrays.asList(ID, TITLE, PLACE, PLACE); private static final String CSV_FIRST_LINE_MUTLIPLE_VALUES = "321, Band 1, Hamburg, Berlin"; private static final String CSV_SECOND_LINE_MUTLIPLE_VALUES = "654, Band 2, Dresden, Hannover"; private static final List CSV_LINES_MUTLIPLE_VALUES = Arrays.asList(CSV_FIRST_LINE_MUTLIPLE_VALUES, @@ -103,7 +102,7 @@ public void shouldPrepareMetadata() throws ImportException { Assert.assertEquals("Wrong number of metadata sets prepared", 3, metadata.size()); Map> metadataSet = metadata.get("123"); Assert.assertNotNull("Metadata for record with ID 123 is null", metadataSet); - Assert.assertEquals("Metadata for record with ID 123 contains wrong title", 2, + Assert.assertEquals("Wrong number of metadata sets prepared", 2, metadataSet.size()); Assert.assertEquals("Metadata for record with ID 123 contains wrong title", "Band 1", metadataSet.get(TITLE).get(0)); diff --git a/Kitodo/src/test/java/org/kitodo/production/services/data/ImportServiceIT.java b/Kitodo/src/test/java/org/kitodo/production/services/data/ImportServiceIT.java index 706ea025da1..8fdcfb705e2 100644 --- a/Kitodo/src/test/java/org/kitodo/production/services/data/ImportServiceIT.java +++ b/Kitodo/src/test/java/org/kitodo/production/services/data/ImportServiceIT.java @@ -190,11 +190,11 @@ public void testImportProcessWithAdditionalMetadata() throws DAOException, Impor HashSet metadata = workpiece.getLogicalStructure().getMetadata(); try { Assert.assertTrue("Process does not contain correct metadata", - assertMetadataSetContainsMetadata(metadata,TITLE,"Band 1")); + assertMetadataSetContainsMetadata(metadata, TITLE, "Band 1")); Assert.assertTrue("Process does not contain correct metadata", - assertMetadataSetContainsMetadata(metadata,PLACE,"Hamburg")); + assertMetadataSetContainsMetadata(metadata, PLACE, "Hamburg")); Assert.assertTrue("Process does not contain correct metadata", - assertMetadataSetContainsMetadata(metadata,PLACE,"Berlin")); + assertMetadataSetContainsMetadata(metadata, PLACE, "Berlin")); } finally { ProcessTestUtils.removeTestProcess(processWithAdditionalMetadata.getId()); } From 7af4367cd920c5fcdb08b4117d1cbc9502d3102b Mon Sep 17 00:00:00 2001 From: BartChris Date: Tue, 9 Apr 2024 19:21:59 +0200 Subject: [PATCH 6/6] correct another assertion message --- .../production/services/catalogimport/MassImportTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Kitodo/src/test/java/org/kitodo/production/services/catalogimport/MassImportTest.java b/Kitodo/src/test/java/org/kitodo/production/services/catalogimport/MassImportTest.java index 4af4abae117..910427e6a7e 100644 --- a/Kitodo/src/test/java/org/kitodo/production/services/catalogimport/MassImportTest.java +++ b/Kitodo/src/test/java/org/kitodo/production/services/catalogimport/MassImportTest.java @@ -123,7 +123,7 @@ public void shouldPrepareMetadata() throws ImportException { metadataSetMultipleValues.containsKey(PLACE)); Assert.assertEquals("Metadata for record with ID 123 has wrong size of place list", 2, metadataSetMultipleValues.get(PLACE).size()); - Assert.assertEquals("Metadata for record with ID 321 has wrong size of place list", "Hamburg", + Assert.assertEquals("Metadata for record with ID 321 contains wrong place", "Hamburg", metadataSetMultipleValues.get(PLACE).get(0)); Assert.assertEquals("Metadata for record with ID 321 contains wrong place", "Berlin", metadataSetMultipleValues.get(PLACE).get(1));