From 8030d71b6728c361ac8c5eb4ce779bc0cb4cd050 Mon Sep 17 00:00:00 2001 From: Arved Solth Date: Wed, 11 Sep 2024 10:24:02 +0200 Subject: [PATCH 1/5] Add EAD collection import --- .../api/schemaconverter/MetadataFormat.java | 1 + .../org/kitodo/constants/StringConstants.java | 12 + .../kitodo/config/enums/ParameterCore.java | 8 +- .../createprocess/CatalogImportDialog.java | 138 ++++- .../createprocess/CreateProcessForm.java | 278 +++++++--- .../forms/createprocess/FileUploadDialog.java | 116 +---- .../createprocess/MetadataImportDialog.java | 3 + .../production/helper/ProcessHelper.java | 110 +++- .../kitodo/production/helper/XMLUtils.java | 109 ++++ .../kitodo/production/model/LazyHitModel.java | 91 +--- .../services/data/ImportService.java | 302 +++++++++-- .../services/data/ProcessService.java | 33 +- .../thread/ImportEadProcessesThread.java | 348 +++++++++++++ .../main/resources/kitodo_config.properties | 7 + .../resources/messages/messages_de.properties | 4 + .../resources/messages/messages_en.properties | 4 + .../webapp/WEB-INF/resources/css/kitodo.css | 11 + .../processFromTemplate/dataEdit.xhtml | 10 +- .../dialogs/eadLevelSelection.xhtml | 62 +++ .../dialogs/errorPopup.xhtml | 2 +- .../dialogs/fileUpload.xhtml | 20 +- .../processFromTemplate/dialogs/import.xhtml | 35 +- .../dialogs/maxNumberOfRecordsExceeded.xhtml | 51 ++ .../dialogs/searchEdit.xhtml | 3 +- .../webapp/pages/processFromTemplate.xhtml | 2 +- .../test/java/org/kitodo/MockDatabase.java | 91 ++++ .../services/data/ImportServiceIT.java | 59 ++- .../resources/importRecords/eadCollection.xml | 238 +++++++++ .../test/resources/rulesets/ruleset_ead.xml | 477 ++++++++++++++++++ Kitodo/src/test/resources/xslt/ead2kitodo.xsl | 255 ++++++++++ .../test/resources/xslt/eadParent2kitodo.xsl | 115 +++++ 31 files changed, 2633 insertions(+), 362 deletions(-) create mode 100644 Kitodo/src/main/java/org/kitodo/production/thread/ImportEadProcessesThread.java create mode 100644 Kitodo/src/main/webapp/WEB-INF/templates/includes/processFromTemplate/dialogs/eadLevelSelection.xhtml create mode 100644 Kitodo/src/main/webapp/WEB-INF/templates/includes/processFromTemplate/dialogs/maxNumberOfRecordsExceeded.xhtml create mode 100644 Kitodo/src/test/resources/importRecords/eadCollection.xml create mode 100755 Kitodo/src/test/resources/rulesets/ruleset_ead.xml create mode 100644 Kitodo/src/test/resources/xslt/ead2kitodo.xsl create mode 100644 Kitodo/src/test/resources/xslt/eadParent2kitodo.xsl diff --git a/Kitodo-API/src/main/java/org/kitodo/api/schemaconverter/MetadataFormat.java b/Kitodo-API/src/main/java/org/kitodo/api/schemaconverter/MetadataFormat.java index 79361cd8084..378c1724b7e 100644 --- a/Kitodo-API/src/main/java/org/kitodo/api/schemaconverter/MetadataFormat.java +++ b/Kitodo-API/src/main/java/org/kitodo/api/schemaconverter/MetadataFormat.java @@ -15,6 +15,7 @@ public enum MetadataFormat { MODS, MARC, PICA, + EAD, OTHER, KITODO; diff --git a/Kitodo-API/src/main/java/org/kitodo/constants/StringConstants.java b/Kitodo-API/src/main/java/org/kitodo/constants/StringConstants.java index 28c61a7e461..71c4d3392c5 100644 --- a/Kitodo-API/src/main/java/org/kitodo/constants/StringConstants.java +++ b/Kitodo-API/src/main/java/org/kitodo/constants/StringConstants.java @@ -21,5 +21,17 @@ public class StringConstants { public static final String DEFAULT_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; // acquisition stages + public static final String CREATE = "create"; public static final String EDIT = "edit"; + + // EAD string constants + public static final String EAD = "ead"; + public static final String LEVEL = "level"; + public static final String C_TAG_NAME = "c"; + // EAD levels + public static final String COLLECTION = "collection"; + public static final String CLASS = "class"; + public static final String SERIES = "series"; + public static final String FILE = "file"; + public static final String ITEM = "item"; } diff --git a/Kitodo/src/main/java/org/kitodo/config/enums/ParameterCore.java b/Kitodo/src/main/java/org/kitodo/config/enums/ParameterCore.java index 40e6a37e765..3aa64a37153 100644 --- a/Kitodo/src/main/java/org/kitodo/config/enums/ParameterCore.java +++ b/Kitodo/src/main/java/org/kitodo/config/enums/ParameterCore.java @@ -655,7 +655,13 @@ public enum ParameterCore implements ParameterInterface { /* Optional parameter can be used to limit the number of processes for which media renaming can be conducted as a * list function. Values different from positive integers are interpreted as "unlimited". */ - MAX_NUMBER_OF_PROCESSES_FOR_MEDIA_RENAMING(new Parameter<>("maxNumberOfProcessesForMediaRenaming", -1)); + MAX_NUMBER_OF_PROCESSES_FOR_MEDIA_RENAMING(new Parameter<>("maxNumberOfProcessesForMediaRenaming", -1)), + + /* + * Optional parameter controlling how many processes are to be displayed and processed in the metadata import mask. + * When more data records are imported the import process is moved to a background task. Default value is 5. + */ + MAX_NUMBER_OF_PROCESSES_FOR_IMPORT_MASK(new Parameter<>("maxNumberOfProcessesForImportMask", 5)); private final Parameter parameter; diff --git a/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/CatalogImportDialog.java b/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/CatalogImportDialog.java index 46a008c8948..c6cabb9e6c6 100644 --- a/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/CatalogImportDialog.java +++ b/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/CatalogImportDialog.java @@ -21,6 +21,7 @@ import javax.faces.context.FacesContext; import javax.xml.parsers.ParserConfigurationException; +import javax.xml.stream.XMLStreamException; import javax.xml.transform.TransformerException; import javax.xml.xpath.XPathExpressionException; @@ -29,7 +30,9 @@ import org.apache.logging.log4j.Logger; import org.kitodo.api.dataeditor.rulesetmanagement.FunctionalMetadata; import org.kitodo.api.externaldatamanagement.SingleHit; +import org.kitodo.api.schemaconverter.DataRecord; import org.kitodo.api.schemaconverter.ExemplarRecord; +import org.kitodo.api.schemaconverter.MetadataFormat; import org.kitodo.data.database.beans.ImportConfiguration; import org.kitodo.data.database.exceptions.DAOException; import org.kitodo.exceptions.CatalogException; @@ -53,7 +56,7 @@ public class CatalogImportDialog extends MetadataImportDialog implements Serializable { private static final Logger logger = LogManager.getLogger(CatalogImportDialog.class); - private final LazyHitModel hitModel = new LazyHitModel(); + private final LazyHitModel hitModel; private static final String ID_PARAMETER_NAME = "ID"; private static final String HITSTABLE_NAME = "hitlistDialogForm:hitlistDialogTable"; @@ -64,6 +67,10 @@ public class CatalogImportDialog extends MetadataImportDialog implements Serial private int numberOfChildren = 0; private String opacErrorMessage = ""; private boolean additionalImport = false; + private String selectedField = ""; + private String searchTerm = ""; + private int importDepth = 2; + /** * Standard constructor. @@ -72,6 +79,7 @@ public class CatalogImportDialog extends MetadataImportDialog implements Serial */ CatalogImportDialog(CreateProcessForm createProcessForm) { super(createProcessForm); + this.hitModel = new LazyHitModel(this); } /** @@ -87,11 +95,11 @@ public void getSelectedRecord() { * @return list of search fields */ public List getSearchFields() { - if (Objects.isNull(hitModel.getImportConfiguration())) { + if (Objects.isNull(createProcessForm.getCurrentImportConfiguration())) { return new LinkedList<>(); } else { try { - return ServiceManager.getImportService().getAvailableSearchFields(hitModel.getImportConfiguration()); + return ServiceManager.getImportService().getAvailableSearchFields(createProcessForm.getCurrentImportConfiguration()); } catch (IllegalArgumentException e) { Helper.setErrorMessage(e.getLocalizedMessage(), logger, e); return new LinkedList<>(); @@ -104,8 +112,8 @@ public List getSearchFields() { */ public void search() { try { - if (skipHitList(hitModel.getImportConfiguration(), hitModel.getSelectedField())) { - getRecordById(hitModel.getSearchTerm()); + if (skipHitList(createProcessForm.getCurrentImportConfiguration(), getSelectedField())) { + getRecordById(getSearchTerm()); } else { List hits = hitModel.load(0, 10, null, SortOrder.ASCENDING, Collections.EMPTY_MAP); if (hits.size() == 1) { @@ -175,16 +183,21 @@ public void getRecordHierarchy() { createProcessForm.setChildProcesses(new LinkedList<>()); int projectId = this.createProcessForm.getProject().getId(); int templateId = this.createProcessForm.getTemplate().getId(); - ImportConfiguration importConfiguration = this.hitModel.getImportConfiguration(); - - // import current and ancestors - LinkedList processes = ServiceManager.getImportService().importProcessHierarchy( - currentRecordId, importConfiguration, projectId, templateId, hitModel.getImportDepth(), - createProcessForm.getRulesetManagement().getFunctionalKeys( - FunctionalMetadata.HIGHERLEVEL_IDENTIFIER)); - // import children - if (this.importChildren) { - importChildren(projectId, templateId, importConfiguration, processes); + ImportConfiguration importConfiguration = createProcessForm.getCurrentImportConfiguration(); + + LinkedList processes; + if (MetadataFormat.EAD.name().equals(importConfiguration.getMetadataFormat())) { + processes = createEadProcesses(importConfiguration); + } else { + // import current and ancestors + processes = ServiceManager.getImportService().importProcessHierarchy(currentRecordId, + importConfiguration, projectId, templateId, getImportDepth(), + createProcessForm.getRulesetManagement().getFunctionalKeys( + FunctionalMetadata.HIGHERLEVEL_IDENTIFIER)); + // import children + if (this.importChildren) { + importChildren(projectId, templateId, importConfiguration, processes); + } } if (!createProcessForm.getProcesses().isEmpty() && additionalImport) { @@ -196,15 +209,39 @@ public void getRecordHierarchy() { attachToExistingParentAndGenerateAtstslIfNotExist(currentTempProcess); showMessageAndRecord(importConfiguration, processes); } + } catch (IOException | ProcessGenerationException | XPathExpressionException | URISyntaxException - | ParserConfigurationException | UnsupportedFormatException | SAXException | DAOException - | ConfigException | TransformerException | NoRecordFoundException | InvalidMetadataValueException - | NoSuchMetadataFieldException e) { + | ParserConfigurationException | UnsupportedFormatException | SAXException | DAOException + | ConfigException | TransformerException | NoRecordFoundException | InvalidMetadataValueException + | NoSuchMetadataFieldException | XMLStreamException e) { throw new CatalogException(e.getLocalizedMessage()); } } } + private LinkedList createEadProcesses(ImportConfiguration importConfiguration) throws NoRecordFoundException, + XPathExpressionException, IOException, ParserConfigurationException, SAXException, XMLStreamException, + UnsupportedFormatException, ProcessGenerationException, URISyntaxException, InvalidMetadataValueException, + TransformerException, NoSuchMetadataFieldException { + LinkedList processes = new LinkedList<>(); + DataRecord externalRecord = ServiceManager.getImportService() + .importExternalDataRecord(importConfiguration, this.currentRecordId, false); + createProcessForm.setXmlString(externalRecord.getOriginalData().toString()); + if (createProcessForm.limitExceeded(externalRecord.getOriginalData().toString())) { + createProcessForm.calculateNumberOfEadElements(); + Ajax.update("maxNumberOfRecordsExceededDialog"); + PrimeFaces.current().executeScript("PF('maxNumberOfRecordsExceededDialog').show();"); + } else { + LinkedList eadProcesses = ServiceManager.getImportService() + .parseImportedEADCollection(externalRecord, importConfiguration, + createProcessForm.getProject().getId(), createProcessForm.getTemplate().getId(), + createProcessForm.getSelectedEadLevel(), createProcessForm.getSelectedParentEadLevel()); + createProcessForm.setChildProcesses(new LinkedList<>(eadProcesses.subList(1, eadProcesses.size() - 1))); + processes = new LinkedList<>(Collections.singletonList(eadProcesses.get(0))); + } + return processes; + } + private void showMessageAndRecord(ImportConfiguration importConfiguration, LinkedList processes) { String summary = Helper.getTranslation("newProcess.catalogueSearch.importSuccessfulSummary"); String detail = Helper.getTranslation("newProcess.catalogueSearch.importSuccessfulDetail", @@ -238,7 +275,7 @@ private void getRecordById(String recordId) { try { if (this.importChildren) { this.numberOfChildren = ServiceManager.getImportService().getNumberOfChildren( - this.hitModel.getImportConfiguration(), this.currentRecordId); + createProcessForm.getCurrentImportConfiguration(), this.currentRecordId); } if (this.importChildren && this.numberOfChildren > NUMBER_OF_CHILDREN_WARNING_THRESHOLD) { Ajax.update("manyChildrenWarningDialog"); @@ -294,7 +331,7 @@ public LinkedList getExemplarRecords() { */ public void setSelectedExemplarRecord(ExemplarRecord selectedExemplarRecord) { try { - ImportService.setSelectedExemplarRecord(selectedExemplarRecord, this.hitModel.getImportConfiguration(), + ImportService.setSelectedExemplarRecord(selectedExemplarRecord, createProcessForm.getCurrentImportConfiguration(), this.createProcessForm.getProcessMetadata().getProcessDetailsElements()); String summary = Helper.getTranslation("newProcess.catalogueSearch.exemplarRecordSelectedSummary"); String detail = Helper.getTranslation("newProcess.catalogueSearch.exemplarRecordSelectedDetail", @@ -303,7 +340,7 @@ public void setSelectedExemplarRecord(ExemplarRecord selectedExemplarRecord) { Ajax.update(FORM_CLIENTID); } catch (ParameterNotFoundException e) { Helper.setErrorMessage("newProcess.catalogueSearch.exemplarRecordParameterNotFoundError", - new Object[] {e.getMessage(), this.hitModel.getImportConfiguration().getTitle() }); + new Object[] {e.getMessage(), createProcessForm.getCurrentImportConfiguration().getTitle() }); } } @@ -314,8 +351,8 @@ public void setSelectedExemplarRecord(ExemplarRecord selectedExemplarRecord) { */ public boolean isParentIdSearchFieldConfigured() { try { - return Objects.nonNull(this.hitModel.getImportConfiguration()) && ServiceManager.getImportService() - .isParentIdSearchFieldConfigured(this.hitModel.getImportConfiguration()); + return Objects.nonNull(createProcessForm.getCurrentImportConfiguration()) && ServiceManager.getImportService() + .isParentIdSearchFieldConfigured(createProcessForm.getCurrentImportConfiguration()); } catch (ConfigException e) { return false; } @@ -360,4 +397,59 @@ public void setAdditionalImport(boolean additionalImport) { this.additionalImport = additionalImport; } + + /** + * Get searchTerm. + * + * @return value of searchTerm + */ + public String getSearchTerm() { + return this.searchTerm; + } + + /** + * Set searchTerm. + * + * @param searchTerm as java.lang.String + */ + public void setSearchTerm(String searchTerm) { + this.searchTerm = searchTerm; + } + + /** + * Get selectedField. + * + * @return value of selectedField + */ + public String getSelectedField() { + return this.selectedField; + } + + /** + * Set selectedField. + * + * @param field as java.lang.String + */ + public void setSelectedField(String field) { + this.selectedField = field; + } + + /** + * Get import depth. + * + * @return import depth + */ + public int getImportDepth() { + return importDepth; + } + + /** + * Set import depth. + * + * @param depth import depth + */ + public void setImportDepth(int depth) { + importDepth = depth; + } + } diff --git a/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/CreateProcessForm.java b/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/CreateProcessForm.java index c9be296e60d..0eaf73f9daa 100644 --- a/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/CreateProcessForm.java +++ b/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/CreateProcessForm.java @@ -12,7 +12,6 @@ package org.kitodo.production.forms.createprocess; import java.io.IOException; -import java.io.OutputStream; import java.net.URI; import java.text.MessageFormat; import java.util.Arrays; @@ -30,24 +29,26 @@ import javax.faces.model.SelectItem; import javax.faces.view.ViewScoped; import javax.inject.Named; +import javax.xml.stream.XMLStreamException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.kitodo.api.MetadataEntry; import org.kitodo.api.dataeditor.rulesetmanagement.FunctionalMetadata; -import org.kitodo.api.dataeditor.rulesetmanagement.MetadataViewInterface; import org.kitodo.api.dataeditor.rulesetmanagement.RulesetManagementInterface; -import org.kitodo.api.dataeditor.rulesetmanagement.SimpleMetadataViewInterface; import org.kitodo.api.dataeditor.rulesetmanagement.StructuralElementViewInterface; -import org.kitodo.api.dataformat.Division; import org.kitodo.api.dataformat.LogicalDivision; import org.kitodo.api.dataformat.Workpiece; import org.kitodo.api.externaldatamanagement.ImportConfigurationType; +import org.kitodo.api.schemaconverter.MetadataFormat; +import org.kitodo.constants.StringConstants; +import org.kitodo.data.database.beans.Client; import org.kitodo.data.database.beans.ImportConfiguration; import org.kitodo.data.database.beans.Process; import org.kitodo.data.database.beans.Project; import org.kitodo.data.database.beans.Ruleset; import org.kitodo.data.database.beans.Template; +import org.kitodo.data.database.beans.User; import org.kitodo.data.database.exceptions.DAOException; import org.kitodo.data.exceptions.DataException; import org.kitodo.exceptions.CommandException; @@ -60,7 +61,10 @@ import org.kitodo.production.enums.ObjectType; import org.kitodo.production.forms.BaseForm; import org.kitodo.production.helper.Helper; +import org.kitodo.production.helper.ProcessHelper; import org.kitodo.production.helper.TempProcess; +import org.kitodo.production.helper.XMLUtils; +import org.kitodo.production.helper.tasks.TaskManager; import org.kitodo.production.interfaces.MetadataTreeTableInterface; import org.kitodo.production.interfaces.RulesetSetupInterface; import org.kitodo.production.metadata.MetadataEditor; @@ -69,6 +73,7 @@ import org.kitodo.production.services.data.ImportService; import org.kitodo.production.services.data.ProcessService; import org.kitodo.production.services.dataeditor.DataEditorService; +import org.kitodo.production.thread.ImportEadProcessesThread; import org.primefaces.PrimeFaces; import org.primefaces.model.TreeNode; @@ -97,9 +102,18 @@ public class CreateProcessForm extends BaseForm implements MetadataTreeTableInte private int progress; private TempProcess currentProcess; private Boolean rulesetConfigurationForOpacImportComplete = null; - private String defaultConfigurationType; + private ImportConfiguration currentImportConfiguration; static final int TITLE_RECORD_LINK_TAB_INDEX = 1; + private final List eadLevels = Arrays.asList(StringConstants.FILE, StringConstants.ITEM); + private final List eadParentLevels = Arrays.asList(StringConstants.COLLECTION, StringConstants.CLASS, + StringConstants.SERIES); + private String selectedEadLevel = StringConstants.FILE; + private String selectedParentEadLevel = StringConstants.COLLECTION; + private String xmlString; + private String filename; + protected int numberOfEadElements; + public CreateProcessForm() { priorityList = ServiceManager.getUserService().getCurrentMetadataLanguage(); } @@ -108,6 +122,10 @@ public CreateProcessForm() { this.priorityList = priorityList; } + public void calculateNumberOfEadElements() throws XMLStreamException { + numberOfEadElements = XMLUtils.getNumberOfElements(xmlString, selectedEadLevel); + } + /** * Returns the ruleset management to access the ruleset. * @@ -429,10 +447,8 @@ public void prepareProcess(int templateId, int projectId, String referringView, project = processGenerator.getProject(); template = processGenerator.getTemplate(); updateRulesetAndDocType(getMainProcess().getRuleset()); - if (Objects.nonNull(project) && Objects.nonNull(project.getDefaultImportConfiguration())) { - setDefaultImportConfiguration(project.getDefaultImportConfiguration()); - } else { - defaultConfigurationType = null; + if (Objects.nonNull(project)) { + setCurrentImportConfiguration(project.getDefaultImportConfiguration()); } if (Objects.nonNull(parentId) && parentId != 0) { ProcessDTO parentProcess = ServiceManager.getProcessService().findById(parentId); @@ -448,43 +464,29 @@ public void prepareProcess(int templateId, int projectId, String referringView, titleRecordLinkTab.setChosenParentProcess(String.valueOf(parentId)); titleRecordLinkTab.chooseParentProcess(); if (Objects.nonNull(project.getDefaultChildProcessImportConfiguration())) { - setDefaultImportConfiguration(project.getDefaultChildProcessImportConfiguration()); - } else { - defaultConfigurationType = null; + setCurrentImportConfiguration(project.getDefaultChildProcessImportConfiguration()); } if (setChildCount(titleRecordLinkTab.getTitleRecordProcess(), rulesetManagement, workpiece)) { updateRulesetAndDocType(getMainProcess().getRuleset()); } } processDataTab.prepare(); - showDefaultImportConfigurationDialog(); + showDialogForImportConfiguration(currentImportConfiguration); } } catch (ProcessGenerationException | DataException | DAOException | IOException e) { Helper.setErrorMessage(e.getLocalizedMessage(), logger, e); } } - private void showDefaultImportConfigurationDialog() { - if (ImportConfigurationType.OPAC_SEARCH.name().equals(defaultConfigurationType)) { - checkRulesetConfiguration(); - } else if (ImportConfigurationType.FILE_UPLOAD.name().equals(defaultConfigurationType)) { - PrimeFaces.current().executeScript("PF('fileUploadDialog').show()"); - } else if (ImportConfigurationType.PROCESS_TEMPLATE.name().equals(defaultConfigurationType)) { - PrimeFaces.current().executeScript("PF('searchEditDialog').show()"); - } - } - - private void setDefaultImportConfiguration(ImportConfiguration importConfiguration) { - defaultConfigurationType = importConfiguration.getConfigurationType(); - if (ImportConfigurationType.OPAC_SEARCH.name().equals(importConfiguration.getConfigurationType())) { - catalogImportDialog.getHitModel().setImportConfiguration(importConfiguration); - PrimeFaces.current().ajax().update("catalogSearchDialog"); - } else if (ImportConfigurationType.PROCESS_TEMPLATE.name().equals(importConfiguration.getConfigurationType())) { - searchDialog.setOriginalProcess(importConfiguration.getDefaultTemplateProcess()); - PrimeFaces.current().ajax().update("searchEditDialog"); - } else if (ImportConfigurationType.FILE_UPLOAD.name().equals(importConfiguration.getConfigurationType())) { - fileUploadDialog.setImportConfiguration(importConfiguration); - PrimeFaces.current().ajax().update("fileUploadDialog"); + private void showDialogForImportConfiguration(ImportConfiguration importConfiguration) { + if (Objects.nonNull(importConfiguration)) { + if (ImportConfigurationType.OPAC_SEARCH.name().equals(importConfiguration.getConfigurationType())) { + PrimeFaces.current().executeScript("PF('catalogSearchDialog').show();"); + } else if (ImportConfigurationType.PROCESS_TEMPLATE.name().equals(importConfiguration.getConfigurationType())) { + PrimeFaces.current().executeScript("PF('searchEditDialog').show();"); + } else if (ImportConfigurationType.FILE_UPLOAD.name().equals(importConfiguration.getConfigurationType())) { + PrimeFaces.current().executeScript("PF('fileUploadDialog').show();"); + } } } @@ -524,7 +526,7 @@ private void createProcessHierarchy() throw new IOException("Unable to create directories for process hierarchy!"); } - if (this.catalogImportDialog.isImportChildren() && !createProcessesLocation(this.childProcesses)) { + if (saveChildProcesses() && !createProcessesLocation(this.childProcesses)) { throw new IOException("Unable to create directories for child processes!"); } saveProcessHierarchyMetadata(); @@ -625,49 +627,11 @@ private void saveProcessHierarchyMetadata() { if (this.processes.indexOf(tempProcess) == 0) { tempProcess.getProcessMetadata().preserve(); } - saveTempProcessMetadata(tempProcess); + ProcessHelper.saveTempProcessMetadata(tempProcess, rulesetManagement, acquisitionStage, priorityList); } // save child processes meta.xml files for (TempProcess tempProcess : this.childProcesses) { - saveTempProcessMetadata(tempProcess); - } - } - - private void saveTempProcessMetadata(TempProcess tempProcess) { - try (OutputStream out = ServiceManager.getFileService() - .write(ServiceManager.getProcessService().getMetadataFileUri(tempProcess.getProcess()))) { - Workpiece workpiece = tempProcess.getWorkpiece(); - workpiece.setId(tempProcess.getProcess().getId().toString()); - if (Objects.nonNull(rulesetManagement)) { - setProcessTitleMetadata(workpiece, tempProcess.getProcess().getTitle()); - } - ServiceManager.getMetsService().save(workpiece, out); - } catch (IOException e) { - Helper.setErrorMessage(e.getLocalizedMessage(), logger, e); - } - } - - private void setProcessTitleMetadata(Workpiece workpiece, String processTitle) { - Collection keysForProcessTitle = rulesetManagement.getFunctionalKeys(FunctionalMetadata.PROCESS_TITLE); - if (!keysForProcessTitle.isEmpty()) { - addAllowedMetadataRecursive(workpiece.getLogicalStructure(), keysForProcessTitle, processTitle); - addAllowedMetadataRecursive(workpiece.getPhysicalStructure(), keysForProcessTitle, processTitle); - } - } - - private void addAllowedMetadataRecursive(Division division, Collection keys, String value) { - StructuralElementViewInterface divisionView = rulesetManagement.getStructuralElementView(division.getType(), - acquisitionStage, priorityList); - for (MetadataViewInterface metadataView : divisionView.getAllowedMetadata()) { - if (metadataView instanceof SimpleMetadataViewInterface && keys.contains(metadataView.getId()) - && division.getMetadata().parallelStream() - .filter(metadata -> metadataView.getId().equals(metadata.getKey())) - .count() < metadataView.getMaxOccurs()) { - MetadataEditor.writeMetadataEntry(division, (SimpleMetadataViewInterface) metadataView, value); - } - } - for (Division child : division.getChildren()) { - addAllowedMetadataRecursive(child, keys, value); + ProcessHelper.saveTempProcessMetadata(tempProcess, rulesetManagement, acquisitionStage, priorityList); } } @@ -849,15 +813,6 @@ public void checkRulesetConfiguration() { PrimeFaces.current().executeScript("PF('recordIdentifierMissingDialog').show();"); } } - - /** - * Get defaultConfigurationType. - * - * @return value of defaultConfigurationType - */ - public String getDefaultConfigurationType() { - return defaultConfigurationType; - } /** * Returns the details of the missing record identifier error. @@ -867,4 +822,159 @@ public String getDefaultConfigurationType() { public Collection getDetailsOfRecordIdentifierMissingError() { return ServiceManager.getImportService().getDetailsOfRecordIdentifierMissingError(); } + + /** + * Set the current import configuration. + * + * @param currentImportConfiguration current import configuration + */ + public void setCurrentImportConfiguration(ImportConfiguration currentImportConfiguration) { + this.currentImportConfiguration = currentImportConfiguration; + if (Objects.nonNull(currentImportConfiguration)) { + if (ImportConfigurationType.OPAC_SEARCH.name().equals(currentImportConfiguration.getConfigurationType())) { + catalogImportDialog.setImportDepth(ImportService.getDefaultImportDepth(currentImportConfiguration)); + catalogImportDialog.setSelectedField(ImportService.getDefaultSearchField(currentImportConfiguration)); + } + else if (ImportConfigurationType.PROCESS_TEMPLATE.name().equals(currentImportConfiguration.getConfigurationType())) { + searchDialog.setOriginalProcess(currentImportConfiguration.getDefaultTemplateProcess()); + } + } + } + + /** + * Get the current ImportConfiguration. + * + * @return current ImportConfiguration + */ + public ImportConfiguration getCurrentImportConfiguration() { + return currentImportConfiguration; + } + + private boolean saveChildProcesses() { + return ((Objects.nonNull(currentImportConfiguration) + && MetadataFormat.EAD.name().equals(currentImportConfiguration.getMetadataFormat())) + || catalogImportDialog.isImportChildren()); + } + + /** + * Get selected ead level. + * + * @return selected ead level + */ + public String getSelectedEadLevel() { + return selectedEadLevel; + } + + /** + * Set selected ead level. + * + * @param selectedEadLevel as String + */ + public void setSelectedEadLevel(String selectedEadLevel) { + this.selectedEadLevel = selectedEadLevel; + } + + /** + * Get selected parent ead level. + * + * @return selected parent ead level + */ + public String getSelectedParentEadLevel() { + return selectedParentEadLevel; + } + + /** + * Set selected parent ead level. + * + * @param selectedParentEadLevel as String + */ + public void setSelectedParentEadLevel(String selectedParentEadLevel) { + this.selectedParentEadLevel = selectedParentEadLevel; + } + + /** + * Get ead levels. + * + * @return ead levels + */ + public List getEadLevels() { + return eadLevels; + } + + /** + * Get parent ead levels. + * + * @return parent ead levels + */ + public List getEadParentLevels() { + return eadParentLevels; + } + + /** + * Get xmlString. + * + * @return xmlString + */ + public String getXmlString() { + return xmlString; + } + + /** + * Set xmlString. + * + * @param xmlString as String + */ + public void setXmlString(String xmlString) { + this.xmlString = xmlString; + } + + /** + * Get filename. + * + * @return filename + */ + public String getFilename() { + return filename; + } + + /** + * Set filename. + * + * @param filename as String + */ + public void setFilename(String filename) { + this.filename = filename; + } + + protected boolean limitExceeded(String xmlString) throws XMLStreamException { + return MetadataFormat.EAD.name().equals(currentImportConfiguration.getMetadataFormat()) + && ServiceManager.getImportService().isMaxNumberOfRecordsExceeded(xmlString, selectedEadLevel); + } + + /** + * Get and return message that informs the user that the maximum number of records that can be processed in the GUI + * has been exceeded. + * + * @return maximum number exceeded message + */ + public String getMaxNumberOfRecordsExceededMessage() { + return ImportService.getMaximumNumberOfRecordsExceededMessage(selectedEadLevel, numberOfEadElements); + } + + /** + * Start background task importing processes from uploaded XML file and redirect either to task manager or desktop, + * depending on user permissions. + * + * @return String containing URL of either task manager or desktop page, depending on user permissions + */ + public String importRecordsInBackground() { + User user = ServiceManager.getUserService().getAuthenticatedUser(); + Client client = ServiceManager.getUserService().getSessionClientOfAuthenticatedUser(); + TaskManager.addTask(new ImportEadProcessesThread(this, user, client)); + if (ServiceManager.getSecurityAccessService().hasAuthorityToViewTaskManagerPage()) { + return "system.jsf?tabIndex=0&faces-redirect=true"; + } else { + return "desktop.jsf?faces-redirect=true"; + } + } } diff --git a/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/FileUploadDialog.java b/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/FileUploadDialog.java index 9874bde64cb..b5252e7949a 100644 --- a/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/FileUploadDialog.java +++ b/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/FileUploadDialog.java @@ -14,21 +14,18 @@ import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.Charset; -import java.util.Collection; import java.util.LinkedList; import java.util.List; import java.util.Objects; import javax.xml.parsers.ParserConfigurationException; +import javax.xml.stream.XMLStreamException; import javax.xml.transform.TransformerException; import javax.xml.xpath.XPathExpressionException; import org.apache.commons.io.IOUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.kitodo.api.dataeditor.rulesetmanagement.FunctionalMetadata; -import org.kitodo.api.schemaconverter.DataRecord; -import org.kitodo.api.schemaconverter.FileFormat; import org.kitodo.api.schemaconverter.MetadataFormat; import org.kitodo.data.database.beans.ImportConfiguration; import org.kitodo.data.database.exceptions.DAOException; @@ -39,17 +36,16 @@ import org.kitodo.exceptions.UnsupportedFormatException; import org.kitodo.production.helper.Helper; import org.kitodo.production.helper.TempProcess; +import org.kitodo.production.helper.XMLUtils; import org.kitodo.production.services.ServiceManager; -import org.kitodo.production.services.data.ImportService; import org.omnifaces.util.Ajax; +import org.primefaces.PrimeFaces; import org.primefaces.event.FileUploadEvent; import org.primefaces.model.file.UploadedFile; -import org.w3c.dom.Document; import org.xml.sax.SAXException; public class FileUploadDialog extends MetadataImportDialog { - private ImportConfiguration importConfiguration; private static final Logger logger = LogManager.getLogger(FileUploadDialog.class); private boolean additionalImport = false; @@ -65,79 +61,39 @@ public FileUploadDialog(CreateProcessForm createProcessForm) { */ public void handleFileUpload(FileUploadEvent event) { UploadedFile uploadedFile = event.getFile(); - ImportService importService = ServiceManager.getImportService(); try { - Document internalDocument = importService.convertDataRecordToInternal( - createRecordFromXMLElement(IOUtils.toString(uploadedFile.getInputStream(), Charset.defaultCharset())), - importConfiguration, false); - TempProcess tempProcess = importService.createTempProcessFromDocument(importConfiguration, internalDocument, - createProcessForm.getTemplate().getId(), createProcessForm.getProject().getId()); - - LinkedList processes = new LinkedList<>(); - processes.add(tempProcess); - - Collection higherLevelIdentifier = this.createProcessForm.getRulesetManagement() - .getFunctionalKeys(FunctionalMetadata.HIGHERLEVEL_IDENTIFIER); - - if (!higherLevelIdentifier.isEmpty()) { - String parentID = importService.getParentID(internalDocument, higherLevelIdentifier.toArray()[0] - .toString(), importConfiguration.getParentElementTrimMode()); - importService.checkForParent(parentID, createProcessForm.getTemplate().getRuleset(), - createProcessForm.getProject().getId()); - if (Objects.isNull(importService.getParentTempProcess())) { - TempProcess parentTempProcess = extractParentRecordFromFile(uploadedFile, internalDocument); - if (Objects.nonNull(parentTempProcess)) { - processes.add(parentTempProcess); - } - } - } - - if (!createProcessForm.getProcesses().isEmpty() && additionalImport) { - extendsMetadataTableOfMetadataTab(processes); + String xmlString = IOUtils.toString(uploadedFile.getInputStream(), Charset.defaultCharset()); + createProcessForm.setXmlString(XMLUtils.removeBom(xmlString)); + createProcessForm.setFilename(uploadedFile.getFileName()); + if (MetadataFormat.EAD.name().equals(createProcessForm.getCurrentImportConfiguration().getMetadataFormat()) + && createProcessForm.limitExceeded(createProcessForm.getXmlString())) { + createProcessForm.calculateNumberOfEadElements(); + Ajax.update("maxNumberOfRecordsExceededDialog"); + PrimeFaces.current().executeScript("PF('maxNumberOfRecordsExceededDialog').show();"); } else { - this.createProcessForm.setProcesses(processes); - TempProcess currentTempProcess = processes.getFirst(); - attachToExistingParentAndGenerateAtstslIfNotExist(currentTempProcess); - createProcessForm.fillCreateProcessForm(currentTempProcess); - Ajax.update(FORM_CLIENTID); + processXmlString(); } } catch (IOException | ProcessGenerationException | URISyntaxException | ParserConfigurationException - | UnsupportedFormatException | SAXException | ConfigException | XPathExpressionException - | TransformerException | DAOException | InvalidMetadataValueException - | NoSuchMetadataFieldException e) { + | UnsupportedFormatException | SAXException | ConfigException | XPathExpressionException + | TransformerException | DAOException | InvalidMetadataValueException | NoSuchMetadataFieldException + | XMLStreamException e) { Helper.setErrorMessage(e.getLocalizedMessage(), logger, e); } } - private TempProcess extractParentRecordFromFile(UploadedFile uploadedFile, Document internalDocument) - throws XPathExpressionException, UnsupportedFormatException, URISyntaxException, IOException, - ParserConfigurationException, SAXException, ProcessGenerationException, TransformerException, - InvalidMetadataValueException, NoSuchMetadataFieldException { - Collection higherLevelIdentifier = this.createProcessForm.getRulesetManagement() - .getFunctionalKeys(FunctionalMetadata.HIGHERLEVEL_IDENTIFIER); - - if (!higherLevelIdentifier.isEmpty()) { - ImportService importService = ServiceManager.getImportService(); - String parentID = importService.getParentID(internalDocument, higherLevelIdentifier.toArray()[0].toString(), - importConfiguration.getParentElementTrimMode()); - if (Objects.nonNull(parentID) && Objects.nonNull(importConfiguration.getParentMappingFile())) { - Document internalParentDocument = importService.convertDataRecordToInternal( - createRecordFromXMLElement(IOUtils.toString(uploadedFile.getInputStream(), Charset.defaultCharset())), - importConfiguration, true); - return importService.createTempProcessFromDocument(importConfiguration, internalParentDocument, - createProcessForm.getTemplate().getId(), createProcessForm.getProject().getId()); - } + private void processXmlString() throws UnsupportedFormatException, XPathExpressionException, + ProcessGenerationException, URISyntaxException, IOException, ParserConfigurationException, SAXException, + InvalidMetadataValueException, TransformerException, NoSuchMetadataFieldException, DAOException { + LinkedList processes = ServiceManager.getImportService().processUploadedFile(createProcessForm); + if (!createProcessForm.getProcesses().isEmpty() && additionalImport) { + extendsMetadataTableOfMetadataTab(processes); + } else { + createProcessForm.setProcesses(processes); + TempProcess currentTempProcess = processes.getFirst(); + attachToExistingParentAndGenerateAtstslIfNotExist(currentTempProcess); + createProcessForm.fillCreateProcessForm(currentTempProcess); + Ajax.update(FORM_CLIENTID); } - return null; - } - - private DataRecord createRecordFromXMLElement(String xmlContent) { - DataRecord record = new DataRecord(); - record.setMetadataFormat( - MetadataFormat.getMetadataFormat(importConfiguration.getMetadataFormat())); - record.setFileFormat(FileFormat.getFileFormat(importConfiguration.getReturnFormat())); - record.setOriginalData(xmlContent); - return record; } @Override @@ -153,24 +109,6 @@ public List getImportConfigurations() { return importConfigurations; } - /** - * Get selected importConfiguration. - * - * @return the selected importConfiguration. - */ - public ImportConfiguration getImportConfiguration() { - return importConfiguration; - } - - /** - * Set selected importConfiguration. - * - * @param importConfiguration the selected catalog. - */ - public void setImportConfiguration(ImportConfiguration importConfiguration) { - this.importConfiguration = importConfiguration; - } - /** * Checks the additional import. * diff --git a/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/MetadataImportDialog.java b/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/MetadataImportDialog.java index 2ae4dcc4617..a57b650056f 100644 --- a/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/MetadataImportDialog.java +++ b/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/MetadataImportDialog.java @@ -54,6 +54,9 @@ public abstract class MetadataImportDialog { */ MetadataImportDialog(CreateProcessForm createProcessForm) { this.createProcessForm = createProcessForm; + this.createProcessForm.numberOfEadElements = 0; + this.createProcessForm.setXmlString(""); + this.createProcessForm.setFilename(""); } void attachToExistingParentAndGenerateAtstslIfNotExist(TempProcess tempProcess) diff --git a/Kitodo/src/main/java/org/kitodo/production/helper/ProcessHelper.java b/Kitodo/src/main/java/org/kitodo/production/helper/ProcessHelper.java index e746851382c..e89ee4684fe 100644 --- a/Kitodo/src/main/java/org/kitodo/production/helper/ProcessHelper.java +++ b/Kitodo/src/main/java/org/kitodo/production/helper/ProcessHelper.java @@ -14,6 +14,7 @@ import static org.kitodo.constants.StringConstants.EDIT; import java.io.IOException; +import java.io.OutputStream; import java.util.Collection; import java.util.HashSet; import java.util.List; @@ -28,10 +29,14 @@ import org.kitodo.api.MetadataEntry; import org.kitodo.api.MetadataGroup; import org.kitodo.api.dataeditor.rulesetmanagement.Domain; +import org.kitodo.api.dataeditor.rulesetmanagement.FunctionalMetadata; import org.kitodo.api.dataeditor.rulesetmanagement.MetadataViewInterface; import org.kitodo.api.dataeditor.rulesetmanagement.RulesetManagementInterface; +import org.kitodo.api.dataeditor.rulesetmanagement.SimpleMetadataViewInterface; import org.kitodo.api.dataeditor.rulesetmanagement.StructuralElementViewInterface; +import org.kitodo.api.dataformat.Division; import org.kitodo.api.dataformat.LogicalDivision; +import org.kitodo.api.dataformat.Workpiece; import org.kitodo.data.database.beans.Process; import org.kitodo.exceptions.InvalidMetadataValueException; import org.kitodo.exceptions.NoSuchMetadataFieldException; @@ -157,9 +162,38 @@ public static void generateAtstslFields(TempProcess tempProcess, List priorityList = ServiceManager.getUserService().getCurrentMetadataLanguage(); + generateAtstslFields(tempProcess, parentTempProcesses, acquisitionStage, priorityList, force); + } + + /** + * Generates TSL/ATS dependent fields for given language list 'priorityList'. + * + * @param tempProcess + * the temp process to generate TSL/ATS dependent fields + * @param parentTempProcesses + * the parent temp processes + * @param acquisitionStage + * current acquisition stage + * @param priorityList + * language list + * @param force + * force regeneration atstsl fields if process title already exists + * @throws ProcessGenerationException + * thrown if process title cannot be created + * @throws InvalidMetadataValueException + * thrown if process workpiece contains invalid metadata + * @throws NoSuchMetadataFieldException + * thrown if process workpiece contains undefined metadata + * @throws IOException + * thrown if ruleset file cannot be loaded + */ + public static void generateAtstslFields(TempProcess tempProcess, List parentTempProcesses, + String acquisitionStage, List priorityList, boolean force) + throws ProcessGenerationException, InvalidMetadataValueException, NoSuchMetadataFieldException, + IOException { RulesetManagementInterface rulesetManagement = ServiceManager.getRulesetService() .openRuleset(tempProcess.getProcess().getRuleset()); - List priorityList = ServiceManager.getUserService().getCurrentMetadataLanguage(); String docType = tempProcess.getWorkpiece().getLogicalStructure().getType(); List processDetails = transformToProcessDetails(tempProcess, rulesetManagement, acquisitionStage, priorityList); @@ -309,6 +343,80 @@ public static void setMetadataDomain(Metadata metadata, RulesetManagementInterfa } } + /** + * Add allowed metadata to given division. + * + * @param division + * division to which allowed metadata is added + * @param keys + * metadata keys to consider + * @param value + * String to be set as value of metadata + * @param rulesetManagement + * RulesetManagementInterface containing metadata rules + * @param acquisitionStage + * current acquisition stage + * @param priorityList + * list of languages + */ + public static void addAllowedMetadataRecursive(Division division, Collection keys, String value, + RulesetManagementInterface rulesetManagement, String acquisitionStage, + List priorityList) { + StructuralElementViewInterface divisionView = rulesetManagement.getStructuralElementView(division.getType(), + acquisitionStage, priorityList); + for (MetadataViewInterface metadataView : divisionView.getAllowedMetadata()) { + if (metadataView instanceof SimpleMetadataViewInterface && keys.contains(metadataView.getId()) + && division.getMetadata().parallelStream() + .filter(metadata -> metadataView.getId().equals(metadata.getKey())) + .count() < metadataView.getMaxOccurs()) { + MetadataEditor.writeMetadataEntry(division, (SimpleMetadataViewInterface) metadataView, value); + } + } + for (Division child : division.getChildren()) { + addAllowedMetadataRecursive(child, keys, value, rulesetManagement, acquisitionStage, priorityList); + } + } + + /** + * Save metadata of given tempProcesses process to meta.xml file. + * + * @param tempProcess + * TempProcess whose Process metadata is saved to a meta.xml file + * @param rulesetManagement + * RulesetManagementInterface containing metadata rules + * @param acquisitionStage + * current acquisition stage + * @param priorityList + * list of languages + */ + public static void saveTempProcessMetadata(TempProcess tempProcess, RulesetManagementInterface rulesetManagement, + String acquisitionStage, List priorityList) { + try (OutputStream out = ServiceManager.getFileService() + .write(ServiceManager.getProcessService().getMetadataFileUri(tempProcess.getProcess()))) { + Workpiece workpiece = tempProcess.getWorkpiece(); + workpiece.setId(tempProcess.getProcess().getId().toString()); + if (Objects.nonNull(rulesetManagement)) { + setProcessTitleMetadata(workpiece, tempProcess.getProcess().getTitle(), rulesetManagement, + acquisitionStage, priorityList); + } + ServiceManager.getMetsService().save(workpiece, out); + } catch (IOException e) { + Helper.setErrorMessage(e.getLocalizedMessage(), logger, e); + } + } + + private static void setProcessTitleMetadata(Workpiece workpiece, String processTitle, + RulesetManagementInterface rulesetManagement, String acquisitionStage, + List priorityList) { + Collection processTitleKeys = rulesetManagement.getFunctionalKeys(FunctionalMetadata.PROCESS_TITLE); + if (!processTitleKeys.isEmpty()) { + ProcessHelper.addAllowedMetadataRecursive(workpiece.getLogicalStructure(), processTitleKeys, processTitle, + rulesetManagement, acquisitionStage, priorityList); + ProcessHelper.addAllowedMetadataRecursive(workpiece.getPhysicalStructure(), processTitleKeys, processTitle, + rulesetManagement, acquisitionStage, priorityList); + } + } + /** * Generate and set the title to process using current title parameter and gets the atstsl. * diff --git a/Kitodo/src/main/java/org/kitodo/production/helper/XMLUtils.java b/Kitodo/src/main/java/org/kitodo/production/helper/XMLUtils.java index 19d3fe12203..8cb21648749 100644 --- a/Kitodo/src/main/java/org/kitodo/production/helper/XMLUtils.java +++ b/Kitodo/src/main/java/org/kitodo/production/helper/XMLUtils.java @@ -15,7 +15,11 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.StringReader; +import java.io.StringWriter; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; import java.util.NoSuchElementException; import java.util.Objects; @@ -23,6 +27,10 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; import javax.xml.transform.OutputKeys; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; @@ -33,9 +41,16 @@ import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; +import org.apache.commons.io.ByteOrderMark; +import org.kitodo.api.schemaconverter.DataRecord; +import org.kitodo.api.schemaconverter.FileFormat; +import org.kitodo.api.schemaconverter.MetadataFormat; +import org.kitodo.constants.StringConstants; +import org.kitodo.data.database.beans.ImportConfiguration; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; +import org.w3c.dom.NodeList; import org.xml.sax.InputSource; import org.xml.sax.SAXException; @@ -164,6 +179,7 @@ public static Document parseXMLString(String xmlString) throws IOException, Pars DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); factory.setNamespaceAware(true); DocumentBuilder builder = factory.newDocumentBuilder(); + xmlString = removeBom(xmlString); return builder.parse(new InputSource(new ByteArrayInputStream(xmlString.getBytes(StandardCharsets.UTF_8)))); } @@ -179,4 +195,97 @@ public static void validateXPathSyntax(String xpathString) throws XPathExpressio XPath xpath = factory.newXPath(); xpath.compile(xpathString); } + + /** + * Remove potential BOM character because XML parser do not handle it properly. + * @param xmlStringWithBom String with potential BOM character + * @return xml String without BOM character + */ + public static String removeBom(String xmlStringWithBom) { + if (Objects.equals(xmlStringWithBom.charAt(0), ByteOrderMark.UTF_BOM)) { + return xmlStringWithBom.substring(1); + } + return xmlStringWithBom; + } + + /** + * Retrieve and return list of XML elements by their tag name, attribute name and attribute value from given + * Document. + * + * @param document XML document from which elements are retrieved + * @param tagName tag name of elements to retrieve + * @param attributeName attribute name of elements to retrieve + * @param attributeValue attribute value of elements to retrieve + * @return list of matching elements + */ + public static List getElementsByTagNameAndAttributeValue(Document document, String tagName, + String attributeName, String attributeValue) { + List elements = new ArrayList<>(); + NodeList nodes = document.getElementsByTagName(tagName); + for (int i = 0; i < nodes.getLength(); i++) { + Element element = (Element) nodes.item(i); + String attributeString = element.getAttribute(attributeName); + if (attributeValue.equals(attributeString)) { + elements.add(element); + } + } + return elements; + } + + /** + * Create and return String representation of given XML element 'Element'. + * + * @param element XML element for which String representation is created and returned + * @return String representation of given XML element 'Element' + * @throws TransformerException when creating String representation of XML element fails + */ + public static String elementToString(Element element) throws TransformerException { + StringWriter stringWriter = new StringWriter(); + Transformer transformer = TransformerFactory.newInstance().newTransformer(); + transformer.transform(new DOMSource(element), new StreamResult(stringWriter)); + return stringWriter.toString(); + } + + /** + * Create DataRecord from given String 'xmlContent' usings settings from given ImportConfiguration + * 'importConfigurations'. + * + * @param xmlContent String containing XML for which a DataRecord is created + * @param importConfiguration ImportConfiguration containing settings to be used for creating DataRecord + * @return DataRecord created for given String 'xmlContent' + */ + public static DataRecord createRecordFromXMLElement(String xmlContent, ImportConfiguration importConfiguration) { + DataRecord record = new DataRecord(); + record.setMetadataFormat( + MetadataFormat.getMetadataFormat(importConfiguration.getMetadataFormat())); + record.setFileFormat(FileFormat.getFileFormat(importConfiguration.getReturnFormat())); + record.setOriginalData(xmlContent); + return record; + } + + /** + * Retrieve and return number of EAD elements with given level "eadLevel", e.g. "" from given + * String "xmlString". + * + * @param xmlString String containing XML to be parsed for elements + * @param eadLevel EAD level of elements to be counted + * @return number of EAD elements with given level + * @throws XMLStreamException when parsing XML string fails + */ + public static int getNumberOfElements(String xmlString, String eadLevel) throws XMLStreamException { + int count = 0; + XMLInputFactory factory = XMLInputFactory.newInstance(); + XMLStreamReader reader = factory.createXMLStreamReader(new StringReader(xmlString)); + while (reader.hasNext()) { + int event = reader.next(); + if (event == XMLStreamConstants.START_ELEMENT) { + if (StringConstants.C_TAG_NAME.equals(reader.getLocalName()) + && eadLevel.equals(reader.getAttributeValue("", StringConstants.LEVEL))) { + count++; + } + } + } + return count; + } + } diff --git a/Kitodo/src/main/java/org/kitodo/production/model/LazyHitModel.java b/Kitodo/src/main/java/org/kitodo/production/model/LazyHitModel.java index 5c60fbaf28b..0ec0878e86c 100644 --- a/Kitodo/src/main/java/org/kitodo/production/model/LazyHitModel.java +++ b/Kitodo/src/main/java/org/kitodo/production/model/LazyHitModel.java @@ -19,25 +19,21 @@ import org.kitodo.api.externaldatamanagement.SearchResult; import org.kitodo.api.externaldatamanagement.SingleHit; -import org.kitodo.data.database.beans.ImportConfiguration; +import org.kitodo.production.forms.createprocess.CatalogImportDialog; import org.kitodo.production.services.ServiceManager; -import org.kitodo.production.services.data.ImportService; import org.primefaces.model.LazyDataModel; import org.primefaces.model.SortOrder; public class LazyHitModel extends LazyDataModel { - private ImportConfiguration importConfiguration; - private String selectedField = ""; - private String searchTerm = ""; - private int importDepth = 2; - + private final CatalogImportDialog catalogImportDialog; private SearchResult searchResult = null; /** * Empty default constructor. Sets default catalog and search field, if configured. */ - public LazyHitModel() { + public LazyHitModel(CatalogImportDialog dialog) { + this.catalogImportDialog = dialog; } @Override @@ -63,7 +59,8 @@ public int getRowCount() { public List load(int first, int resultSize, String sortField, SortOrder sortOrder, Map filters) { searchResult = ServiceManager.getImportService().performSearch( - this.selectedField, this.searchTerm, this.importConfiguration, first, resultSize); + catalogImportDialog.getSelectedField(), catalogImportDialog.getSearchTerm(), + catalogImportDialog.createProcessForm.getCurrentImportConfiguration(), first, resultSize); if (Objects.isNull(searchResult) || Objects.isNull(searchResult.getHits())) { return Collections.emptyList(); @@ -83,80 +80,4 @@ public List getHits() { return Collections.emptyList(); } } - - /** - * Getter for importConfiguration. - * - * @return value of importConfiguration - */ - public ImportConfiguration getImportConfiguration() { - return importConfiguration; - } - - /** - * Setter for importConfiguration. This also sets the catalogs default search field, if configured. - * - * @param importConfiguration ImportConfiguration - */ - public void setImportConfiguration(ImportConfiguration importConfiguration) { - this.importConfiguration = importConfiguration; - if (Objects.nonNull(importConfiguration)) { - this.setSelectedField(ImportService.getDefaultSearchField(importConfiguration)); - this.setImportDepth(ServiceManager.getImportService().getDefaultImportDepth(importConfiguration)); - } - } - - /** - * Get searchTerm. - * - * @return value of searchTerm - */ - public String getSearchTerm() { - return this.searchTerm; - } - - /** - * Set searchTerm. - * - * @param searchTerm as java.lang.String - */ - public void setSearchTerm(String searchTerm) { - this.searchTerm = searchTerm; - } - - /** - * Get selectedField. - * - * @return value of selectedField - */ - public String getSelectedField() { - return this.selectedField; - } - - /** - * Set selectedField. - * - * @param field as java.lang.String - */ - public void setSelectedField(String field) { - this.selectedField = field; - } - - /** - * Get import depth. - * - * @return import depth - */ - public int getImportDepth() { - return importDepth; - } - - /** - * Set import depth. - * - * @param depth import depth - */ - public void setImportDepth(int depth) { - importDepth = depth; - } } diff --git a/Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java b/Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java index 3ae4e898572..44a2704f91a 100644 --- a/Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java +++ b/Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java @@ -35,6 +35,7 @@ import javax.faces.model.SelectItem; import javax.xml.parsers.ParserConfigurationException; +import javax.xml.stream.XMLStreamException; import javax.xml.transform.TransformerException; import javax.xml.xpath.XPath; import javax.xml.xpath.XPathConstants; @@ -66,6 +67,7 @@ import org.kitodo.config.ConfigCore; import org.kitodo.config.ConfigProject; import org.kitodo.config.enums.ParameterCore; +import org.kitodo.constants.StringConstants; import org.kitodo.data.database.beans.ImportConfiguration; import org.kitodo.data.database.beans.MappingFile; import org.kitodo.data.database.beans.Process; @@ -92,6 +94,7 @@ import org.kitodo.exceptions.RecordIdentifierMissingDetail; import org.kitodo.exceptions.UnsupportedFormatException; import org.kitodo.production.dto.ProcessDTO; +import org.kitodo.production.forms.createprocess.CreateProcessForm; import org.kitodo.production.forms.createprocess.ProcessBooleanMetadata; import org.kitodo.production.forms.createprocess.ProcessDetail; import org.kitodo.production.forms.createprocess.ProcessFieldedMetadata; @@ -282,7 +285,7 @@ else if (SearchInterfaceType.OAI.name().equals(configuration.getInterfaceType()) * @param importConfiguration ImportConfiguration * @return default import depth of given import configuration */ - public int getDefaultImportDepth(ImportConfiguration importConfiguration) { + public static int getDefaultImportDepth(ImportConfiguration importConfiguration) { int depth = importConfiguration.getDefaultImportDepth(); if (depth < 0 || depth > 5) { return 2; @@ -409,8 +412,7 @@ public String getParentID(Document document, String higherLevelIdentifier, Strin */ public TempProcess createTempProcessFromDocument(ImportConfiguration importConfiguration, Document document, int templateID, int projectID) - throws ProcessGenerationException, IOException, TransformerException, InvalidMetadataValueException, - NoSuchMetadataFieldException { + throws ProcessGenerationException, IOException, TransformerException { Process process = null; // "processGenerator" needs to be initialized when function is called for the first time if (Objects.isNull(processGenerator)) { @@ -443,7 +445,8 @@ private String importProcessAndReturnParentID(String recordId, LinkedList processes, String parentID, Template template, String parentIdMetadata) throws ProcessGenerationException, IOException, XPathExpressionException, ParserConfigurationException, - NoRecordFoundException, UnsupportedFormatException, URISyntaxException, SAXException, DAOException, + NoRecordFoundException, UnsupportedFormatException, URISyntaxException, SAXException, InvalidMetadataValueException, NoSuchMetadataFieldException { int level = 1; this.parentTempProcess = null; @@ -589,21 +613,46 @@ private void importParents(String recordId, ImportConfiguration importConfigurat /** * Check if there already is a parent process in Database. */ - public void checkForParent(String parentID, Ruleset ruleset, int projectID) - throws DAOException, IOException, ProcessGenerationException { - if (Objects.isNull(parentID)) { - this.parentTempProcess = null; - return; + public void checkForParent(String parentID, Ruleset ruleset, int projectID) { + this.parentTempProcess = retrieveParentTempProcess(parentID, ruleset, projectID); + } + + /** + * Get parentTempProcess. + * + * @return value of parentTempProcess + */ + public TempProcess getParentTempProcess() { + return parentTempProcess; + } + + /** + * Retrieve temp process containing process with given 'parentRecordId' as functional metadata 'recordIdentifier'. + * + * @param parentRecordId 'recordIdentifier' value of parent process + * @param ruleset Ruleset containing metadata rules + * @param projectID ID of project to which process belongs + * @return TempProcess containing parent process if it exists in database or null otherwise + */ + public TempProcess retrieveParentTempProcess(String parentRecordId, Ruleset ruleset, int projectID) { + if (Objects.isNull(parentRecordId) || Objects.isNull(ruleset)) { + logger.info("Unable to get parent temp process: parentRecordId or ruleset is null!"); + return null; } - Process parentProcess = loadParentProcess(ruleset, projectID, parentID); - if (Objects.nonNull(parentProcess)) { - logger.info("Linking last imported process to parent process with ID {} in database!", parentID); + Process parentProcess; + try { + parentProcess = loadParentProcess(ruleset, projectID, parentRecordId); + if (Objects.isNull(parentProcess)) { + return null; + } URI workpieceUri = ServiceManager.getProcessService().getMetadataFileUri(parentProcess); Workpiece parentWorkpiece = ServiceManager.getMetsService().loadWorkpiece(workpieceUri); - this.parentTempProcess = new TempProcess(parentProcess, parentWorkpiece); - return; + return new TempProcess(parentProcess, parentWorkpiece); + } catch (ProcessGenerationException | DAOException | IOException e) { + logger.error("Error retrieving parent process with 'recordIdentifier' {}, project ID {} and ruleset {}", + parentRecordId, projectID, ruleset.getTitle()); + return null; } - this.parentTempProcess = null; } private List searchChildRecords(ImportConfiguration config, String parentId, int numberOfRows) { @@ -679,11 +728,23 @@ public LinkedList getChildProcesses(ImportConfiguration importConfi } } - private Document importDocument(ImportConfiguration importConfiguration, String identifier, - boolean extractExemplars, boolean isParentInRecord) - throws NoRecordFoundException, UnsupportedFormatException, URISyntaxException, IOException, - XPathExpressionException, ParserConfigurationException, SAXException, ProcessGenerationException { - // ################ IMPORT ################# + /** + * Retrieve data from external data source and return DataRecord containing said external data. + * + * @param importConfiguration ImportConfiguration used for data import + * @param identifier ID of record to be loaded from external source + * @param extractExemplars boolean flag signaling whether exemplar records should be extracted from data record + * @return DataRecord containing data loaded from external source + * @throws NoRecordFoundException when loading full record by ID fails + * @throws IOException when extracting exemplar records fails + * @throws XPathExpressionException when extracting exemplar records fails + * @throws ParserConfigurationException when extracting exemplar records fails + * @throws SAXException when extracting exemplar records fails + */ + public DataRecord importExternalDataRecord(ImportConfiguration importConfiguration, String identifier, + boolean extractExemplars) + throws NoRecordFoundException, IOException, + XPathExpressionException, ParserConfigurationException, SAXException { importModule = initializeImportModule(); DataRecord dataRecord = importModule.getFullRecordById( createDataImportFromImportConfiguration(importConfiguration), @@ -691,7 +752,92 @@ private Document importDocument(ImportConfiguration importConfiguration, String if (extractExemplars) { exemplarRecords = extractExemplarRecords(dataRecord, importConfiguration); } - return convertDataRecordToInternal(dataRecord, importConfiguration, isParentInRecord); + return dataRecord; + } + + /** + * This method transforms a given data record that contains an EAD collection as an XML string into a list of + * temp processes. The first temp process in the list will contain the 'collection' itself, while all following temp + * processes contain the 'item' level child records of the collection. + * + * @param importedEADRecord XML string representation of + * @return list of temp processes + */ + public LinkedList parseImportedEADCollection(DataRecord importedEADRecord, + ImportConfiguration importConfiguration, int projectId, + int templateId, String eadChildProcessLevel, + String eadParentProcessLevel) + throws IOException, ParserConfigurationException, SAXException, ProcessGenerationException, + TransformerException, UnsupportedFormatException, XPathExpressionException, URISyntaxException, + InvalidMetadataValueException, NoSuchMetadataFieldException { + LinkedList eadCollectionProcesses = new LinkedList<>(); + + Document eadCollectionDocument = XMLUtils.parseXMLString((String) importedEADRecord.getOriginalData()); + + List parentElements = getEADElements(eadCollectionDocument, eadParentProcessLevel); + List childElements = getEADElements(eadCollectionDocument, eadChildProcessLevel); + + if (parentElements.size() != 1) { + throw new ProcessGenerationException( + String.format("EAD XML does not contain exactly one element of parent level '%s'!", + eadParentProcessLevel)); + } + + // create temp processes for parent (e.g. "collection") and children (e.g. "files") + TempProcess collectionProcess = createTempProcessFromElement(parentElements.get(0), importConfiguration, + projectId, templateId, true); + eadCollectionProcesses.add(collectionProcess); + + List parentProcesses = Collections.singletonList(collectionProcess); + for (Element fileElement : childElements) { + TempProcess currentFileProcess = createTempProcessFromElement(fileElement, importConfiguration, + projectId, templateId, false); + ProcessHelper.generateAtstslFields(currentFileProcess, parentProcesses, ACQUISITION_STAGE_CREATE, false); + eadCollectionProcesses.add(currentFileProcess); + } + + return eadCollectionProcesses; + } + + /** + * Create and return TempProcess from given XML element 'Element'. + * + * @param element XML element to be transformed into TempProcess + * @param importConfiguration ImportConfiguration containing settings used to transform XML element to TempProcess + * @param projectId ID of project for which process is created + * @param templateId ID of template with which process is created + * @param isParent boolean flag signaling whether the process to be created is a parent process and thus a separate + * parentMappingFile should be used for the XML transformation or not + * @return TempProcess created from given XML element + * @throws TransformerException when parsing external data failed + * @throws UnsupportedFormatException when external data could not be transformed to internal metadata format + * @throws XPathExpressionException when external data could not be transformed to internal metadata format + * @throws ProcessGenerationException when external data could not be transformed to internal metadata format + * @throws URISyntaxException when external data could not be transformed to internal metadata format + * @throws IOException when creating new process failed + * @throws ParserConfigurationException when external data could not be transformed to internal metadata format + * @throws SAXException when external data could not be transformed to internal metadata format + */ + public TempProcess createTempProcessFromElement(Element element, ImportConfiguration importConfiguration, + int projectId, int templateId, boolean isParent) + throws TransformerException, UnsupportedFormatException, XPathExpressionException, + ProcessGenerationException, URISyntaxException, IOException, ParserConfigurationException, SAXException { + String collectionString = XMLUtils.elementToString(element); + DataRecord externalCollectionRecord = XMLUtils.createRecordFromXMLElement(collectionString, + importConfiguration); + Document internalEadCollectionDocument = convertDataRecordToInternal(externalCollectionRecord, + importConfiguration, isParent); + return createTempProcessFromDocument(importConfiguration, internalEadCollectionDocument, templateId, projectId); + } + + private List getEADElements(Document document, String level) { + return XMLUtils.getElementsByTagNameAndAttributeValue(document, StringConstants.C_TAG_NAME, + StringConstants.LEVEL, level); + } + + public boolean isMaxNumberOfRecordsExceeded(String xmlString, String tagName) throws XMLStreamException { + int numberOfRecords = XMLUtils.getNumberOfElements(xmlString, tagName); + return numberOfRecords > ConfigCore.getIntParameterOrDefaultValue(ParameterCore.MAX_NUMBER_OF_PROCESSES_FOR_IMPORT_MASK); } /** @@ -951,15 +1097,6 @@ public static void setSelectedExemplarRecord(ExemplarRecord exemplarRecord, Impo } } - /** - * Get parentTempProcess. - * - * @return value of parentTempProcess - */ - public TempProcess getParentTempProcess() { - return parentTempProcess; - } - private Process loadParentProcess(Ruleset ruleset, int projectId, String parentId) throws ProcessGenerationException, DAOException, IOException { @@ -1184,8 +1321,10 @@ public static void processTempProcess(TempProcess tempProcess, RulesetManagement ProcessHelper.generateAtstslFields(tempProcess, processDetails, parentTempProcesses, docType, rulesetManagement, acquisitionStage, priorityList); - if (!ProcessValidator.isProcessTitleCorrect(tempProcess.getProcess().getTitle())) { - throw new ProcessGenerationException("Unable to create process"); + String processTitle = tempProcess.getProcess().getTitle(); + if (!ProcessValidator.isProcessTitleCorrect(processTitle)) { + throw new ProcessGenerationException(String.format("Unable to create process (invalid process title '%s')", + processTitle)); } Process process = tempProcess.getProcess(); @@ -1511,4 +1650,95 @@ public static boolean userMayLinkToParent(int processId) throws DAOException { return processInAssignedProject(processId) || ServiceManager.getSecurityAccessService().hasAuthorityToLinkToProcessesOfUnassignedProjects(); } + + /** + * Get and return message informing user that the max number of processes that can be handled in the GUI at the same + * time has been exceeded and that the import will therefore be delegated to a background task. + * + * @param processLevelElement EAD level of elements that are being imported as processes + * @param numberOfElements number of processes that are to be imported + * @return max number exceeded message + */ + public static String getMaximumNumberOfRecordsExceededMessage(String processLevelElement, int numberOfElements) { + return Helper.getTranslation("createProcessForm.limitExceeded", + String.valueOf(ConfigCore.getIntParameterOrDefaultValue(ParameterCore + .MAX_NUMBER_OF_PROCESSES_FOR_IMPORT_MASK)), + String.valueOf(numberOfElements), + processLevelElement); + } + + /** + * Create list of TempProcess from XML string stored in given CreateProcessForm 'createProcessForm'. + * + * @param createProcessForm CreateProcessForm containing XML string to convert into TempProcesses + * @return list of TempProcess created from XML string stored in given CreateProcessForm + * @throws ProcessGenerationException when converting XML string to TempProcesses fails + * @throws IOException when creating TempProcesses fails + * @throws InvalidMetadataValueException when creating EAD processes from XML string fails + * @throws TransformerException when creating TempProcesses fails + * @throws NoSuchMetadataFieldException when creating EAD processes from XML string fails + * @throws UnsupportedFormatException when converting XML string to TempProcesses fails + * @throws XPathExpressionException when creating TempProcesses fails + * @throws ParserConfigurationException when creating TempProcesses fails + * @throws URISyntaxException when creating TempProcesses fails + * @throws SAXException when creating TempProcesses fails + */ + public LinkedList processUploadedFile(CreateProcessForm createProcessForm) + throws ProcessGenerationException, IOException, InvalidMetadataValueException, TransformerException, + NoSuchMetadataFieldException, UnsupportedFormatException, XPathExpressionException, + ParserConfigurationException, URISyntaxException, SAXException { + LinkedList processes = new LinkedList<>(); + ImportConfiguration importConfiguration = createProcessForm.getCurrentImportConfiguration(); + DataRecord externalRecord = XMLUtils.createRecordFromXMLElement(createProcessForm.getXmlString(), + importConfiguration); + if (MetadataFormat.EAD.name().equals(importConfiguration.getMetadataFormat())) { + LinkedList eadProcesses = parseImportedEADCollection(externalRecord, importConfiguration, + createProcessForm.getProject().getId(), createProcessForm.getTemplate().getId(), + createProcessForm.getSelectedEadLevel(), createProcessForm.getSelectedParentEadLevel()); + createProcessForm.setChildProcesses(new LinkedList<>(eadProcesses.subList(1, eadProcesses.size()))); + processes = new LinkedList<>(Collections.singletonList(eadProcesses.get(0))); + } else { + Document internalDocument = convertDataRecordToInternal(externalRecord, importConfiguration, false); + TempProcess tempProcess = createTempProcessFromDocument(importConfiguration, internalDocument, + createProcessForm.getTemplate().getId(), createProcessForm.getProject().getId()); + processes.add(tempProcess); + Collection higherLevelIdentifier = createProcessForm.getRulesetManagement() + .getFunctionalKeys(FunctionalMetadata.HIGHERLEVEL_IDENTIFIER); + if (!higherLevelIdentifier.isEmpty()) { + String parentID = getParentID(internalDocument, higherLevelIdentifier.toArray()[0] + .toString(), importConfiguration.getParentElementTrimMode()); + checkForParent(parentID, createProcessForm.getTemplate().getRuleset(), + createProcessForm.getProject().getId()); + if (Objects.isNull(getParentTempProcess())) { + TempProcess parentTempProcess = extractParentRecordFromFile(internalDocument, createProcessForm); + if (Objects.nonNull(parentTempProcess)) { + processes.add(parentTempProcess); + } + } + } + } + return processes; + } + + private TempProcess extractParentRecordFromFile(Document internalDocument, CreateProcessForm createProcessForm) + throws XPathExpressionException, UnsupportedFormatException, URISyntaxException, IOException, + ParserConfigurationException, SAXException, ProcessGenerationException, TransformerException { + Collection higherLevelIdentifier = createProcessForm.getRulesetManagement() + .getFunctionalKeys(FunctionalMetadata.HIGHERLEVEL_IDENTIFIER); + + if (!higherLevelIdentifier.isEmpty()) { + ImportConfiguration importConfiguration = createProcessForm.getCurrentImportConfiguration(); + ImportService importService = ServiceManager.getImportService(); + String parentID = importService.getParentID(internalDocument, higherLevelIdentifier.toArray()[0].toString(), + importConfiguration.getParentElementTrimMode()); + if (Objects.nonNull(parentID) && Objects.nonNull(importConfiguration.getParentMappingFile())) { + Document internalParentDocument = importService.convertDataRecordToInternal( + XMLUtils.createRecordFromXMLElement(createProcessForm.getXmlString(), importConfiguration), + importConfiguration, true); + return importService.createTempProcessFromDocument(importConfiguration, internalParentDocument, + createProcessForm.getTemplate().getId(), createProcessForm.getProject().getId()); + } + } + return null; + } } diff --git a/Kitodo/src/main/java/org/kitodo/production/services/data/ProcessService.java b/Kitodo/src/main/java/org/kitodo/production/services/data/ProcessService.java index 90fa915528b..2163414941c 100644 --- a/Kitodo/src/main/java/org/kitodo/production/services/data/ProcessService.java +++ b/Kitodo/src/main/java/org/kitodo/production/services/data/ProcessService.java @@ -316,10 +316,10 @@ public void saveToIndex(Process process, boolean forceRefresh) * Find all parent processes for a process ordered such that the root parent comes first. * * @param process the process whose parents are to be found - * @return the list of parent processes (direct parents and grand parents, and more) + * @return the list of parent processes (direct parents and grandparents, and more) */ public List findParentProcesses(Process process) { - List parents = new ArrayList(); + List parents = new ArrayList<>(); Process current = process; while (Objects.nonNull(current.getParent())) { current = current.getParent(); @@ -958,18 +958,7 @@ public ProcessDTO convertJSONObjectToDTO(Map jsonObject, boolean convertLastProcessingDates(jsonObject, processDTO); convertTaskProgress(jsonObject, processDTO); - List> jsonArray = ProcessTypeField.PROPERTIES.getJsonArray(jsonObject); - List properties = new ArrayList<>(); - for (Map stringObjectMap : jsonArray) { - PropertyDTO propertyDTO = new PropertyDTO(); - Object title = stringObjectMap.get(JSON_TITLE); - Object value = stringObjectMap.get(JSON_VALUE); - if (Objects.nonNull(title)) { - propertyDTO.setTitle(title.toString()); - propertyDTO.setValue(Objects.nonNull(value) ? value.toString() : ""); - properties.add(propertyDTO); - } - } + List properties = getProperties(jsonObject); processDTO.setProperties(properties); if (!related) { @@ -985,6 +974,22 @@ public ProcessDTO convertJSONObjectToDTO(Map jsonObject, boolean return processDTO; } + private List getProperties(Map jsonObject) throws DataException { + List> jsonArray = ProcessTypeField.PROPERTIES.getJsonArray(jsonObject); + List properties = new ArrayList<>(); + for (Map stringObjectMap : jsonArray) { + PropertyDTO propertyDTO = new PropertyDTO(); + Object title = stringObjectMap.get(JSON_TITLE); + Object value = stringObjectMap.get(JSON_VALUE); + if (Objects.nonNull(title)) { + propertyDTO.setTitle(title.toString()); + propertyDTO.setValue(Objects.nonNull(value) ? value.toString() : ""); + properties.add(propertyDTO); + } + } + return properties; + } + /** * Parses last processing dates from the jsonObject and adds them to the processDTO bean. diff --git a/Kitodo/src/main/java/org/kitodo/production/thread/ImportEadProcessesThread.java b/Kitodo/src/main/java/org/kitodo/production/thread/ImportEadProcessesThread.java new file mode 100644 index 00000000000..0b8ec0abe88 --- /dev/null +++ b/Kitodo/src/main/java/org/kitodo/production/thread/ImportEadProcessesThread.java @@ -0,0 +1,348 @@ +/* + * (c) Kitodo. Key to digital objects e. V. + * + * This file is part of the Kitodo project. + * + * It is licensed under GNU General Public License version 3 or later. + * + * For the full copyright and license information, please read the + * GPL3-License.txt file that was distributed with this source code. + */ + +package org.kitodo.production.thread; + +import static org.kitodo.constants.StringConstants.CREATE; + +import java.io.IOException; +import java.io.StringReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Objects; + +import javax.xml.namespace.QName; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.stream.XMLEventReader; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.events.Attribute; +import javax.xml.stream.events.EndElement; +import javax.xml.stream.events.Namespace; +import javax.xml.stream.events.StartElement; +import javax.xml.stream.events.XMLEvent; +import javax.xml.transform.TransformerException; +import javax.xml.xpath.XPathExpressionException; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.kitodo.api.Metadata; +import org.kitodo.api.MetadataEntry; +import org.kitodo.api.dataeditor.rulesetmanagement.FunctionalMetadata; +import org.kitodo.api.dataeditor.rulesetmanagement.RulesetManagementInterface; +import org.kitodo.constants.StringConstants; +import org.kitodo.data.database.beans.Client; +import org.kitodo.data.database.beans.ImportConfiguration; +import org.kitodo.data.database.beans.User; +import org.kitodo.data.database.exceptions.DAOException; +import org.kitodo.data.exceptions.DataException; +import org.kitodo.exceptions.CommandException; +import org.kitodo.exceptions.InvalidMetadataValueException; +import org.kitodo.exceptions.NoSuchMetadataFieldException; +import org.kitodo.exceptions.ProcessGenerationException; +import org.kitodo.exceptions.UnsupportedFormatException; +import org.kitodo.production.forms.createprocess.CreateProcessForm; +import org.kitodo.production.helper.ProcessHelper; +import org.kitodo.production.helper.TempProcess; +import org.kitodo.production.helper.XMLUtils; +import org.kitodo.production.helper.tasks.EmptyTask; +import org.kitodo.production.metadata.MetadataEditor; +import org.kitodo.production.security.SecurityUserDetails; +import org.kitodo.production.services.ServiceManager; +import org.kitodo.production.services.data.ImportService; +import org.kitodo.production.services.data.ProcessService; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContext; +import org.springframework.security.core.context.SecurityContextHolder; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + + +public class ImportEadProcessesThread extends EmptyTask { + + private static final Logger logger = LogManager.getLogger(ImportEadProcessesThread.class); + private final ImportService importService = ServiceManager.getImportService(); + private final String xmlString; + private final String eadLevel; + private final String eadParentLevel; + private final int projectId; + private final int templateId; + private final ImportConfiguration importConfiguration; + private final RulesetManagementInterface rulesetManagementInterface; + private final List priorityList; + private final List namespaces; + private final User user; + private final Client client; + private TempProcess parentProcess = null; + private int count; + + /** + * Standard constructor, creating instance of ImportEadProcessesThread with settings from given + * CreateProcessForm 'createProcessForm' and for given User 'user' and Client 'client'. + * + * @param createProcessForm CreateProcessForm instance encapsulating various settings required for this thread + * @param user User instance to which this thread is assigned + * @param client Client instance of current user + */ + public ImportEadProcessesThread(CreateProcessForm createProcessForm, User user, Client client) { + super(createProcessForm.getFilename()); + this.xmlString = createProcessForm.getXmlString(); + this.eadLevel = createProcessForm.getSelectedEadLevel(); + this.eadParentLevel = createProcessForm.getSelectedParentEadLevel(); + this.projectId = createProcessForm.getProject().getId(); + this.templateId = createProcessForm.getTemplate().getId(); + this.importConfiguration = createProcessForm.getCurrentImportConfiguration(); + this.rulesetManagementInterface = createProcessForm.getRulesetManagement(); + this.priorityList = ServiceManager.getUserService().getCurrentMetadataLanguage(); + this.namespaces = new ArrayList<>(); + this.user = user; + this.client = client; + } + + @Override + protected void setNameDetail(String detail) { + super.setNameDetail(detail); + } + + @Override + public void run() { + setAuthenticatedUser(); + List newProcessIds = new ArrayList<>(); + int newParentId = 0; + try { + int numberOfElements = XMLUtils.getNumberOfElements(xmlString, eadLevel); + XMLInputFactory inputFactory = XMLInputFactory.newInstance(); + XMLEventReader eventReader = inputFactory.createXMLEventReader(new StringReader(xmlString)); + boolean inProcessElement = false; + boolean inParentProcessElement = false; + String currentTagName; + StringBuilder stringBuilder = new StringBuilder(); + while (eventReader.hasNext()) { + + XMLEvent event = eventReader.nextEvent(); + switch (event.getEventType()) { + case XMLEvent.START_ELEMENT: + StartElement startElement = event.asStartElement(); + currentTagName = startElement.getName().getLocalPart(); + if (StringConstants.EAD.equals(currentTagName)) { + Iterator namespaceIterator = startElement.getNamespaces(); + while (namespaceIterator.hasNext()) { + namespaces.add(namespaceIterator.next()); + } + } + if (StringConstants.C_TAG_NAME.equals(currentTagName)) { + // all metadata until first nested "" element is considered when creating parent process + // for EAD "collection" + if (inParentProcessElement) { + inParentProcessElement = false; + stringBuilder.append(""); + TempProcess tempProcess = parseXmlStringToTempProcess(stringBuilder.toString(), true); + TempProcess existingParent = getParentCandidate(tempProcess); + // distinguish between existing "collection" parent and newly created one + if (Objects.isNull(existingParent)) { + parentProcess = processTempProcess(tempProcess); + newParentId = parentProcess.getProcess().getId(); + } else { + parentProcess = existingParent; + } + stringBuilder = new StringBuilder(); + } + Attribute levelAttribute = startElement.getAttributeByName(QName.valueOf(StringConstants + .LEVEL)); + Attribute idAttribute = startElement.getAttributeByName(QName.valueOf("id")); + if (Objects.nonNull(levelAttribute) && Objects.nonNull(idAttribute)) { + if (eadLevel.equals(levelAttribute.getValue())) { + inProcessElement = true; + count++; + stringBuilder.append(processStartElement(event)); + } else { + if (eadParentLevel.equals(levelAttribute.getValue())) { + inParentProcessElement = true; + stringBuilder.append(processStartElement(event)); + } + } + } + } else { + if (inParentProcessElement || inProcessElement) { + String content = event.toString(); + if (StringUtils.isNotBlank(content)) { + stringBuilder.append(removeDefaultNamespaceUri(content)); + } + } + } + break; + case XMLEvent.END_ELEMENT: + EndElement endElement = event.asEndElement(); + String endElementName = endElement.getName().getLocalPart(); + if (inProcessElement && StringConstants.C_TAG_NAME.equals(endElementName)) { + int progress = (count * 100) / numberOfElements; + setProgress(progress); + inProcessElement = false; + String content = event.toString(); + stringBuilder.append(removeDefaultNamespaceUri(content)); + newProcessIds.add(parseXmlStringToProcessedTempProcess(stringBuilder.toString()).getProcess().getId()); + stringBuilder = new StringBuilder(); + } else { + if (inParentProcessElement || inProcessElement) { + String content = event.toString(); + if (StringUtils.isNotBlank(content)) { + stringBuilder.append(removeDefaultNamespaceUri(content)); + } + } + } + break; + default: + if (inParentProcessElement || inProcessElement) { + String content = event.toString(); + if (StringUtils.isNotBlank(content)) { + stringBuilder.append(removeDefaultNamespaceUri(content)); + } + } + break; + } + } + + } catch (XMLStreamException | IOException | ParserConfigurationException | SAXException + | UnsupportedFormatException | XPathExpressionException | ProcessGenerationException + | URISyntaxException | InvalidMetadataValueException | TransformerException + | NoSuchMetadataFieldException | DataException | CommandException e) { + logger.error(e.getMessage(), e); + cleanUpProcesses(newProcessIds, newParentId); + throw new RuntimeException(e); + } + } + + private void cleanUpProcesses(List processIds, int newParentId) { + logger.error("Deleting processes created until this point to resolve erroneous intermediate state."); + // cleanup any processes that might have been created until this point + for (int id : processIds) { + try { + ProcessService.deleteProcess(ServiceManager.getProcessService().getById(id)); + } catch (DataException | DAOException | IOException ex) { + logger.error(ex); + } + } + if (newParentId > 0 && Objects.nonNull(parentProcess) && Objects.nonNull(parentProcess.getProcess())) { + try { + ProcessService.deleteProcess(parentProcess.getProcess()); + } catch (DataException | IOException ex) { + logger.error(ex); + } + } + } + + private String processStartElement(XMLEvent event) { + String content = event.toString(); + for (Namespace namespace : namespaces) { + String prefix = namespace.getPrefix(); + if (StringUtils.isNotBlank(prefix)) { + content = content.replace(">", " xmlns:" + namespace.getPrefix() + "=\"" + + namespace.getNamespaceURI() + "\">"); + } + } + return removeDefaultNamespaceUri(content); + } + + // This method is a workaround for a problem encountered during development of the EAD import where XML files + // defining a default names space without namespace prefix could be parsed incorrectly. + private String removeDefaultNamespaceUri(String xmlEventString) { + for (Namespace namespace : namespaces) { + String prefix = namespace.getPrefix(); + if (StringUtils.isBlank(prefix)) { + return xmlEventString.replace("['" + namespace.getNamespaceURI() + "']::", ""); + } + } + return xmlEventString; + } + + private TempProcess getParentCandidate(TempProcess tempProcess) { + String recordId = getRecordIdentifier(tempProcess); + if (Objects.nonNull(recordId)) { + TempProcess parent = ServiceManager.getImportService().retrieveParentTempProcess(recordId, + tempProcess.getProcess().getRuleset(), projectId); + if (Objects.nonNull(parent)) { + return parent; + } + } + return null; + } + + private TempProcess processTempProcess(TempProcess tempProcess) throws ProcessGenerationException, IOException, + InvalidMetadataValueException, NoSuchMetadataFieldException, DataException, CommandException { + ProcessHelper.generateAtstslFields(tempProcess, Collections.emptyList(), CREATE, priorityList, false); + tempProcess.getProcessMetadata().preserve(); + ImportService.processTempProcess(tempProcess, rulesetManagementInterface, CREATE, priorityList, parentProcess); + saveTempProcessMetadata(tempProcess); + if (Objects.nonNull(parentProcess)) { + ProcessService.setParentRelations(parentProcess.getProcess(), tempProcess.getProcess()); + MetadataEditor.addLink(parentProcess.getProcess(), String.valueOf(count - 1), tempProcess.getProcess() + .getId()); + ServiceManager.getProcessService().save(tempProcess.getProcess(), true); + } + return tempProcess; + } + + // used to parse parent (e.g. "collection") + private TempProcess parseXmlStringToTempProcess(String xmlString, boolean isParent) throws IOException, + ParserConfigurationException, SAXException, UnsupportedFormatException, XPathExpressionException, + ProcessGenerationException, URISyntaxException, TransformerException { + Document elementDocument = XMLUtils.parseXMLString(xmlString); + Element element = elementDocument.getDocumentElement(); + return importService.createTempProcessFromElement(element, importConfiguration, projectId, templateId, isParent); + } + + // used to parse children (e.g. "files") + private TempProcess parseXmlStringToProcessedTempProcess(String xmlElementString) throws IOException, + ParserConfigurationException, SAXException, UnsupportedFormatException, XPathExpressionException, + ProcessGenerationException, URISyntaxException, InvalidMetadataValueException, TransformerException, + NoSuchMetadataFieldException, DataException, CommandException { + TempProcess tempProcess = parseXmlStringToTempProcess(xmlElementString, false); + return processTempProcess(tempProcess); + } + + private String getRecordIdentifier(TempProcess tempProcess) { + Collection recordIdMetadata = rulesetManagementInterface + .getFunctionalKeys(FunctionalMetadata.RECORD_IDENTIFIER); + for (String recordId : recordIdMetadata) { + for (Metadata metadata : tempProcess.getWorkpiece().getLogicalStructure().getMetadata()) { + if (metadata instanceof MetadataEntry && recordId.equals(metadata.getKey())) { + return ((MetadataEntry) metadata).getValue(); + } + } + } + return null; + } + + private void setAuthenticatedUser() { + SecurityUserDetails securityUserDetails = new SecurityUserDetails(user); + Authentication authentication = new UsernamePasswordAuthenticationToken(securityUserDetails, null, + securityUserDetails.getAuthorities()); + SecurityContext securityContext = SecurityContextHolder.getContext(); + securityUserDetails.setSessionClient(client); + securityContext.setAuthentication(authentication); + } + + private void saveTempProcessMetadata(TempProcess tempProcess) throws DataException, IOException, CommandException { + ServiceManager.getProcessService().save(tempProcess.getProcess(), true); + URI processBaseUri = ServiceManager.getFileService().createProcessLocation(tempProcess.getProcess()); + tempProcess.getProcess().setProcessBaseUri(processBaseUri); + ProcessHelper.saveTempProcessMetadata(tempProcess, rulesetManagementInterface, CREATE, priorityList); + } +} diff --git a/Kitodo/src/main/resources/kitodo_config.properties b/Kitodo/src/main/resources/kitodo_config.properties index 3a658ba8b2e..99a9a591a82 100644 --- a/Kitodo/src/main/resources/kitodo_config.properties +++ b/Kitodo/src/main/resources/kitodo_config.properties @@ -789,3 +789,10 @@ security.secret.ldapManagerPassword= # This optional parameter can be used to limit the number of processes for which media renaming can be conducted as a # list function. Values different from positive integers are interpreted as "unlimited". #maxNumberOfProcessesForMediaRenaming=10000 + +# The parameter 'maxNumberOfProcessesForHierarchicalImport' is used to limit the number of processes during the +# hierarchical import that are loaded into the metadata import dialog at the same time. When this limit is exceeded, +# e.g. more data records are imported (from a search interface or from an uploaded EAD file), a dialog is displayed +# informing the user that the processes will be imported using a background task and the progress can be tracked via the +# task manager. +maxNumberOfProcessesForImportMask=5 diff --git a/Kitodo/src/main/resources/messages/messages_de.properties b/Kitodo/src/main/resources/messages/messages_de.properties index 7ae616e7aaf..4fe1f4ab5a8 100644 --- a/Kitodo/src/main/resources/messages/messages_de.properties +++ b/Kitodo/src/main/resources/messages/messages_de.properties @@ -663,6 +663,7 @@ importingData=Daten werden importiert... importChildren=Kindvorg\u00E4nge importieren importDepth=Importtiefe importDms=Export in das DMS +importEadProcessesThread=EAD-Bestands-Import importProcesses=Vorg\u00E4nge importieren imprint=Impressum # used in "LegalTexts.java" @@ -841,6 +842,8 @@ newProcess.catalogueSearch.savingSecondaryProcesses=Speichern von untergeordnete newProcess.childProcesses=Kindvorg\u00E4nge newProcess.docTypeMetadataMissing=Der Datensatz konnte nicht importiert werden. Es wurde kein Metadatum f\u00fcr die Dokumenttypklassifizierung ("use='docType'") im verwendeten Regelsatz "{0}" gefunden! newProcess.fileUpload.heading=Dateiupload +newProcess.fileUpload.eadLevelParent=EAD-Level \u00DCberordnung +newProcess.fileUpload.eadLevelChildren=EAD-Level Unterordnung newProcess.processHierarchy=Vorgangshierarchie newProcess.rulesetSelection.deactivated=Der Regelsatz wird durch die verwendete Produktionsvorlage vorgegeben und kann w\u00E4hrend der Vorgangserstellung nicht ge\u00E4ndert werden newProcess.titleGeneration.creationRuleNotFound=Es konnte keine Regel zur Titelgenerierung f\u00fcr Vorg\u00E4nge vom Typ \u201E{0}\u201C im Regelsatz \u201E{1}\u201C gefunden werden! @@ -970,6 +973,7 @@ createProcessForm.titleRecordLinkTab.noInsertionPosition=Sie k\u00F6nnen den Vor createProcessForm.titleRecordLinkTab.searchButtonClick.empty=Suchanfrage leer createProcessForm.titleRecordLinkTab.searchButtonClick.error=Die Suche lief auf einen Fehler: createProcessForm.titleRecordLinkTab.searchButtonClick.noHits=Die Suche ist abgeschlossen, aber es wurden keine potentiellen Elternvorg\u00E4nge gefunden. Sie k\u00F6nnen nur Vorg\u00E4nge verlinken, die denselben Regelsatz nutzen. +createProcessForm.limitExceeded=Sie k\u00F6nnen maximal {0} Datens\u00E4tze gleichzeitig \u00FCber diese Importmaske verarbeiten. Die hochgeladene Datei \u00FCberschreitet dieses Limit und enth\u00E4lt {1} Datens\u00E4tze vom Typ "{2}". Die Verarbeitung der Datei wird daher in eine Hintergrundaufgabe verschoben, deren Fortschritt Sie mit entsprechender Berechtigung \u00FCber den Taskmanager nachverfolgen k\u00F6nnen. quarter=Quartal quarters=Quartale ready=Fertig diff --git a/Kitodo/src/main/resources/messages/messages_en.properties b/Kitodo/src/main/resources/messages/messages_en.properties index ddd626b3b0a..e47ad13a226 100644 --- a/Kitodo/src/main/resources/messages/messages_en.properties +++ b/Kitodo/src/main/resources/messages/messages_en.properties @@ -663,6 +663,7 @@ importingData=Data are being imported importChildren=Import child processes importDepth=Import depth importDms=Import into DMS +importEadProcessesThread=EAD collection import importProcesses=Import processes imprint=Imprint # used in "LegalTexts.java" @@ -841,6 +842,8 @@ newProcess.catalogueSearch.savingSecondaryProcesses=Saving secondary processes newProcess.childProcesses=Child processes newProcess.docTypeMetadataMissing=Unable to import metadata. No metadata for document type classification ("use='docType'") found in ruleset "{0}"! newProcess.fileUpload.heading=Fileupload +newProcess.fileUpload.eadLevelParent=EAD-Level parent process +newProcess.fileUpload.eadLevelChildren=EAD-Level child processes newProcess.processHierarchy=Process hierarchy newProcess.rulesetSelection.deactivated=Ruleset is predetermined by given template and cannot be changed during process creation newProcess.titleGeneration.creationRuleNotFound=No title creation rules found for docType \u201E{0}\u201C in ruleset \u201E{1}\u201C! @@ -970,6 +973,7 @@ createProcessForm.titleRecordLinkTab.noInsertionPosition=You cannot link the pro createProcessForm.titleRecordLinkTab.searchButtonClick.empty=Query is empty createProcessForm.titleRecordLinkTab.searchButtonClick.error=The search ran on an error: createProcessForm.titleRecordLinkTab.searchButtonClick.noHits=The search completed, but nothing was found. You can only find processes that have the same rule set and are potential parent processes. +createProcessForm.limitExceeded=The maximum number of records to be processed in the import mask is {0}. The uploaded file exceeds this limit and contains {1} elements of type "{2}". Process import will be delegated to a background task. quarter=quarter quarters=quarters ready=Ready diff --git a/Kitodo/src/main/webapp/WEB-INF/resources/css/kitodo.css b/Kitodo/src/main/webapp/WEB-INF/resources/css/kitodo.css index ad558a59e50..be9ec7189f7 100644 --- a/Kitodo/src/main/webapp/WEB-INF/resources/css/kitodo.css +++ b/Kitodo/src/main/webapp/WEB-INF/resources/css/kitodo.css @@ -2401,6 +2401,17 @@ form#metadata div label, margin: 0; } +.metadata-import-dialog-form .ead-level-selection-table { + width: 100%; +} + +.metadata-import-dialog-form .ead-level-selection-table > tbody > tr > td.ui-panelgrid-cell { + vertical-align: top; +} + +.ui-dialog .ui-panelgrid-cell div.ead-level-selection div { + margin-bottom: auto; +} #metadataAccordion\:metadata .ui-treetable-indent, #metadataAccordion\:metadata .ui-treetable-toggler { diff --git a/Kitodo/src/main/webapp/WEB-INF/templates/includes/processFromTemplate/dataEdit.xhtml b/Kitodo/src/main/webapp/WEB-INF/templates/includes/processFromTemplate/dataEdit.xhtml index 5f27134438e..973ce390f4f 100644 --- a/Kitodo/src/main/webapp/WEB-INF/templates/includes/processFromTemplate/dataEdit.xhtml +++ b/Kitodo/src/main/webapp/WEB-INF/templates/includes/processFromTemplate/dataEdit.xhtml @@ -33,10 +33,10 @@ + @(.carousel-button)"/> + @(.carousel-button)"/> diff --git a/Kitodo/src/main/webapp/WEB-INF/templates/includes/processFromTemplate/dialogs/eadLevelSelection.xhtml b/Kitodo/src/main/webapp/WEB-INF/templates/includes/processFromTemplate/dialogs/eadLevelSelection.xhtml new file mode 100644 index 00000000000..b082a32492b --- /dev/null +++ b/Kitodo/src/main/webapp/WEB-INF/templates/includes/processFromTemplate/dialogs/eadLevelSelection.xhtml @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Kitodo/src/main/webapp/WEB-INF/templates/includes/processFromTemplate/dialogs/errorPopup.xhtml b/Kitodo/src/main/webapp/WEB-INF/templates/includes/processFromTemplate/dialogs/errorPopup.xhtml index 153f5ffc5fa..00bb8997a7b 100644 --- a/Kitodo/src/main/webapp/WEB-INF/templates/includes/processFromTemplate/dialogs/errorPopup.xhtml +++ b/Kitodo/src/main/webapp/WEB-INF/templates/includes/processFromTemplate/dialogs/errorPopup.xhtml @@ -28,7 +28,7 @@

- +

diff --git a/Kitodo/src/main/webapp/WEB-INF/templates/includes/processFromTemplate/dialogs/fileUpload.xhtml b/Kitodo/src/main/webapp/WEB-INF/templates/includes/processFromTemplate/dialogs/fileUpload.xhtml index d9288c2a5cc..97c1f01a693 100644 --- a/Kitodo/src/main/webapp/WEB-INF/templates/includes/processFromTemplate/dialogs/fileUpload.xhtml +++ b/Kitodo/src/main/webapp/WEB-INF/templates/includes/processFromTemplate/dialogs/fileUpload.xhtml @@ -19,14 +19,15 @@ xmlns:p="http://primefaces.org/ui">

#{msgs['newProcess.fileUpload.heading']}

- +
@@ -36,7 +37,7 @@ autoWidth="false" required="#{not empty param['catalogSearchForm:performCatalogSearch']}" immediate="true" - value="#{CreateProcessForm.fileUploadDialog.importConfiguration}" + value="#{CreateProcessForm.currentImportConfiguration}" converter="#{importConfigurationConverter}"> - +
@@ -57,7 +58,7 @@
- + + + + +

#{msgs['newProcess.catalogueSearch.heading']}

- +
@@ -36,7 +37,7 @@ autoWidth="false" required="#{not empty param['catalogSearchForm:performCatalogSearch']}" immediate="true" - value="#{CreateProcessForm.catalogImportDialog.hitModel.importConfiguration}" + value="#{CreateProcessForm.currentImportConfiguration}" converter="#{importConfigurationConverter}"> + catalogSearchForm:catalogSearchButton, + catalogSearchForm:eadLevelSelectionWrapper"/>
@@ -62,9 +64,9 @@ value="#{msgs.field}"/> + value="#{CreateProcessForm.catalogImportDialog.selectedField}"> @@ -75,8 +77,8 @@ value="#{msgs.importDepth}"/>
@@ -91,8 +93,8 @@ value="#{msgs.value}"/> @@ -108,15 +110,21 @@
+ + + + + + * + * This file is part of the Kitodo project. + * + * It is licensed under GNU General Public License version 3 or later. + * + * For the full copyright and license information, please read the + * GPL3-License.txt file that was distributed with this source code. + * +--> + + + + + + + + + + + + diff --git a/Kitodo/src/main/webapp/WEB-INF/templates/includes/processFromTemplate/dialogs/searchEdit.xhtml b/Kitodo/src/main/webapp/WEB-INF/templates/includes/processFromTemplate/dialogs/searchEdit.xhtml index ccdb6282b86..98d7125dc76 100644 --- a/Kitodo/src/main/webapp/WEB-INF/templates/includes/processFromTemplate/dialogs/searchEdit.xhtml +++ b/Kitodo/src/main/webapp/WEB-INF/templates/includes/processFromTemplate/dialogs/searchEdit.xhtml @@ -22,7 +22,7 @@ id="searchEditDialog" width="578" modal="true" - visible="#{CreateProcessForm.defaultConfigurationType eq 'PROCESS_TEMPLATE'}" + visible="#{false}" appendTo="@(body)" resizable="false">

#{msgs.processTemplate}

@@ -58,6 +58,7 @@ icon="fa fa-download" iconPos="right"/> - + diff --git a/Kitodo/src/test/java/org/kitodo/MockDatabase.java b/Kitodo/src/test/java/org/kitodo/MockDatabase.java index 014b36bb0b3..8a83f92497c 100644 --- a/Kitodo/src/test/java/org/kitodo/MockDatabase.java +++ b/Kitodo/src/test/java/org/kitodo/MockDatabase.java @@ -750,6 +750,97 @@ public static Map insertProcessesForHierarchyTests() throws DAO return testProcesses; } + private static ImportConfiguration insertEadImportConfiguration() throws DAOException, DataException { + // EAD mapping files (for "file" and "collection" level) + MappingFile eadMappingFile = new MappingFile(); + eadMappingFile.setInputMetadataFormat(MetadataFormat.EAD.name()); + eadMappingFile.setOutputMetadataFormat(MetadataFormat.KITODO.name()); + eadMappingFile.setFile("ead2kitodo.xsl"); + eadMappingFile.setTitle("EAD to Kitodo"); + ServiceManager.getMappingFileService().saveToDatabase(eadMappingFile); + + MappingFile eadParentMappingFile = new MappingFile(); + eadParentMappingFile.setInputMetadataFormat(MetadataFormat.EAD.name()); + eadParentMappingFile.setOutputMetadataFormat(MetadataFormat.KITODO.name()); + eadParentMappingFile.setFile("eadParent2kitodo.xsl"); + eadParentMappingFile.setTitle("EAD Parent to Kitodo"); + ServiceManager.getMappingFileService().saveToDatabase(eadParentMappingFile); + + // EAD upload import configuration + ImportConfiguration eadUploadConfiguration = new ImportConfiguration(); + eadUploadConfiguration.setTitle("EAD upload configuration"); + eadUploadConfiguration.setConfigurationType(ImportConfigurationType.FILE_UPLOAD.name()); + eadUploadConfiguration.setMetadataFormat(MetadataFormat.EAD.name()); + eadUploadConfiguration.setReturnFormat(FileFormat.XML.name()); + eadUploadConfiguration.setMappingFiles(Collections.singletonList(eadMappingFile)); + eadUploadConfiguration.setParentMappingFile(eadParentMappingFile); + ServiceManager.getImportConfigurationService().saveToDatabase(eadUploadConfiguration); + return eadUploadConfiguration; + } + + private static Template insertEadTemplate(Ruleset eadRuleset, Project eadImportProject, Client client) + throws DAOException, DataException { + Task firstTask = new Task(); + firstTask.setTitle("Open"); + firstTask.setOrdering(1); + firstTask.setRepeatOnCorrection(true); + firstTask.setEditType(TaskEditType.MANUAL_SINGLE); + firstTask.setProcessingStatus(TaskStatus.OPEN); + + Task secondTask = new Task(); + secondTask.setTitle("Locked"); + secondTask.setOrdering(2); + secondTask.setRepeatOnCorrection(true); + secondTask.setEditType(TaskEditType.MANUAL_SINGLE); + secondTask.setTypeImagesWrite(true); + secondTask.setProcessingStatus(TaskStatus.LOCKED); + + List tasks = Arrays.asList(firstTask, secondTask); + + // EAD template + Template eadTemplate = new Template(); + eadTemplate.setTitle("EAD template"); + eadTemplate.setRuleset(eadRuleset); + eadTemplate.getProjects().add(eadImportProject); + eadTemplate.setClient(client); + ServiceManager.getTemplateService().save(eadTemplate); + eadTemplate.setTasks(tasks); + Role role = ServiceManager.getRoleService().getById(1); + for (Task task : eadTemplate.getTasks()) { + task.setTemplate(eadTemplate); + task.getRoles().add(role); + role.getTasks().add(task); + ServiceManager.getTaskService().save(task); + } + return eadTemplate; + } + + public static Project insertProjectForEadImport(User user, Client client) throws DAOException, DataException { + + // EAD ruleset + Ruleset eadRuleset = new Ruleset(); + eadRuleset.setTitle("EAD ruleset"); + eadRuleset.setFile("ruleset_ead.xml"); + eadRuleset.setClient(client); + ServiceManager.getRulesetService().save(eadRuleset); + + ImportConfiguration eadUploadConfiguration = insertEadImportConfiguration(); + + // EAD project + Project eadImportProject = new Project(); + eadImportProject.setTitle("EAD test project"); + eadImportProject.getUsers().add(user); + eadImportProject.setClient(client); + eadImportProject.setDefaultImportConfiguration(eadUploadConfiguration); + ServiceManager.getProjectService().save(eadImportProject); + + Template eadTemplate = insertEadTemplate(eadRuleset, eadImportProject, client); + + eadImportProject.getTemplates().add(eadTemplate); + ServiceManager.getProjectService().save(eadImportProject); + return eadImportProject; + } + /** * Insert ruleset. * @param rulesetTitle ruleset title diff --git a/Kitodo/src/test/java/org/kitodo/production/services/data/ImportServiceIT.java b/Kitodo/src/test/java/org/kitodo/production/services/data/ImportServiceIT.java index bf28dfde4f0..7c7fc7651a9 100644 --- a/Kitodo/src/test/java/org/kitodo/production/services/data/ImportServiceIT.java +++ b/Kitodo/src/test/java/org/kitodo/production/services/data/ImportServiceIT.java @@ -18,6 +18,8 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.kitodo.constants.StringConstants.COLLECTION; +import static org.kitodo.constants.StringConstants.FILE; import com.xebialabs.restito.server.StubServer; @@ -59,8 +61,10 @@ import org.kitodo.api.schemaconverter.MetadataFormat; import org.kitodo.config.ConfigCore; import org.kitodo.config.enums.ParameterCore; +import org.kitodo.data.database.beans.Client; import org.kitodo.data.database.beans.ImportConfiguration; import org.kitodo.data.database.beans.Process; +import org.kitodo.data.database.beans.Project; import org.kitodo.data.database.beans.Ruleset; import org.kitodo.data.database.beans.Template; import org.kitodo.data.database.beans.UrlParameter; @@ -73,12 +77,14 @@ import org.kitodo.exceptions.NoSuchMetadataFieldException; import org.kitodo.exceptions.ProcessGenerationException; import org.kitodo.exceptions.UnsupportedFormatException; +import org.kitodo.production.forms.createprocess.CreateProcessForm; import org.kitodo.production.forms.createprocess.ProcessDetail; import org.kitodo.production.forms.createprocess.ProcessTextMetadata; import org.kitodo.production.helper.ProcessHelper; import org.kitodo.production.helper.TempProcess; import org.kitodo.production.helper.XMLUtils; import org.kitodo.production.services.ServiceManager; +import org.kitodo.production.thread.ImportEadProcessesThread; import org.kitodo.test.utils.ProcessTestUtils; import org.kitodo.test.utils.TestConstants; import org.w3c.dom.Document; @@ -94,6 +100,7 @@ public class ImportServiceIT { private static final ImportService importService = ServiceManager.getImportService(); private static StubServer server; private static final String TEST_FILE_PATH = "src/test/resources/sruTestRecord.xml"; + private static final String EAD_COLLECTION_FILE = "importRecords/eadCollection.xml"; private static final String TEST_FILE_PATH_NUMBER_OF_HITS = "src/test/resources/importRecords/sruResponseNumberOfHits.xml"; private static final String TEST_FILE_SUCCESS_RESPONSE_PATH = "src/test/resources/customInterfaceSuccessResponse.xml"; private static final String TEST_FILE_ERROR_RESPONSE_PATH = "src/test/resources/customInterfaceErrorResponse.xml"; @@ -296,12 +303,11 @@ public void shouldCreateTempProcessFromDocument() throws Exception { * Tests whether parent process with provided process ID exists and created parent TempProcess from it. * * @throws DAOException when test ruleset cannot be loaded from database - * @throws ProcessGenerationException when checking for parent process fails * @throws IOException when checking for parent process fails * @throws DataException when copying test metadata file fails */ @Test - public void shouldCheckForParent() throws DAOException, ProcessGenerationException, IOException, DataException { + public void shouldCheckForParent() throws DAOException, IOException, DataException { int parentTestId = MockDatabase.insertTestProcess("Test parent process", PROJECT_ID, TEMPLATE_ID, RULESET_ID); ProcessTestUtils.copyTestMetadataFile(parentTestId, TEST_KITODO_METADATA_FILE); Ruleset ruleset = ServiceManager.getRulesetService().getById(RULESET_ID); @@ -553,6 +559,55 @@ public void shouldAddProperties() throws Exception { } } + /** + * Test EAD import. + * + * @throws Exception when something goes wrong + */ + @Test + public void shouldImportEadCollection() throws Exception { + User user = ServiceManager.getUserService().getById(1); + Client client = ServiceManager.getClientService().getById(1); + Project eadProject = MockDatabase.insertProjectForEadImport(user, client); + Template eadTemplate = eadProject.getTemplates().get(0); + CreateProcessForm createProcessForm = new CreateProcessForm(); + createProcessForm.setProject(eadProject); + createProcessForm.setTemplate(eadTemplate); + createProcessForm.setSelectedEadLevel(FILE); + createProcessForm.setSelectedParentEadLevel(COLLECTION); + createProcessForm.setCurrentImportConfiguration(eadProject.getDefaultImportConfiguration()); + createProcessForm.updateRulesetAndDocType(eadTemplate.getRuleset()); + File script = new File(ConfigCore.getParameter(ParameterCore.SCRIPT_CREATE_DIR_META)); + List allIds = ServiceManager.getProcessService().findAllIDs(); + if (!SystemUtils.IS_OS_WINDOWS) { + ExecutionPermission.setExecutePermission(script); + } + try (InputStream inputStream = Thread.currentThread().getContextClassLoader() + .getResourceAsStream(EAD_COLLECTION_FILE)) { + if (Objects.nonNull(inputStream)) { + String xmlString = IOUtils.toString(inputStream, Charset.defaultCharset()); + createProcessForm.setXmlString(xmlString); + ImportEadProcessesThread eadProcessesThread = new ImportEadProcessesThread(createProcessForm, user, client); + eadProcessesThread.start(); + assertTrue(eadProcessesThread.isAlive(), "Process should have been started"); + eadProcessesThread.join(3000); + assertFalse(eadProcessesThread.isAlive(), "Process should have been stopped"); + } + } + if (!SystemUtils.IS_OS_WINDOWS) { + ExecutionPermission.setNoExecutePermission(script); + } + List allIdsWithEad = ServiceManager.getProcessService().findAllIDs(); + // EAD test file contains one collection and 5 files, so the system should contain 6 new processes altogether + assertEquals(allIds.size() + 6, allIdsWithEad.size(), + "Database does not contain the correct number of processes after EAD import"); + if (allIdsWithEad.removeAll(allIds)) { + for (int processId : allIdsWithEad) { + ProcessTestUtils.removeTestProcess(processId); + } + } + } + private String getProcessDetailByMetadataId(String metadataId, List processDetails) { for (ProcessDetail processDetail : processDetails) { if (Objects.equals(processDetail.getMetadataID(), metadataId) && processDetail instanceof ProcessTextMetadata) { diff --git a/Kitodo/src/test/resources/importRecords/eadCollection.xml b/Kitodo/src/test/resources/importRecords/eadCollection.xml new file mode 100644 index 00000000000..f818b31508d --- /dev/null +++ b/Kitodo/src/test/resources/importRecords/eadCollection.xml @@ -0,0 +1,238 @@ + + + + + + Test-Archiv + + + + + + + + Custom Collection + Test-Bestand des Test-Archivs + 1989-1991 + + Foto + 0,3 lfm. + + + + + + + + Beschreibung +

Li Europan lingues es membres del sam familie. Lor separat existentie es un myth. Por scientie, musica, + sport etc, litot Europa usa li sam vocabular. Li lingues differe solmen in li grammatica, li pronunciation + e li plu commun vocabules. Omnicos directe al desirabilite de un nov lingua franca

+ + + Zugangsbeschränkung +

extern

+ + + Literatur +

Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor.

+ + + + Martin Mustermann + + + + + Personen + + + + + Custom ID 101 + Testtitel 1 + 1. Januar 1950 - 31. Dezember 1953 + Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo + ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur + ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa + quis enim. Donec. + Ankauf + + Foto + 1 Bild + + + +

+ + + + Technische Daten / Farbigkeit +

s/w

+ + + Technische Daten / Fototyp +

Positiv

+ + + + Deutschland + + + Müller, Hansi + + + + + + Custom ID 102 + Testtitel 2 + 1. Januar 1950 - 31. Dezember 1953 + Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo + ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur + ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa + quis enim. Donec. + Ankauf + + Foto + 1 Bild + + + +

+ + + + Technische Daten / Farbigkeit +

s/w

+ + + Technische Daten / Fototyp +

Positiv

+ + + + Deutschland + + + Müller, Hansi + + + + + + Custom ID 103 + Testtitel 3 + 1. Januar 1950 - 31. Dezember 1953 + Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo + ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur + ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa + quis enim. Donec. + Ankauf + + Foto + 1 Bild + + + +

+ + + + Technische Daten / Farbigkeit +

s/w

+ + + Technische Daten / Fototyp +

Positiv

+ + + + Deutschland + + + Müller, Hansi + + + + + + Custom ID 104 + Testtitel 4 + 1. Januar 1950 - 31. Dezember 1953 + Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo + ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur + ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa + quis enim. Donec. + Ankauf + + Foto + 1 Bild + + + +

+ + + + Technische Daten / Farbigkeit +

s/w

+ + + Technische Daten / Fototyp +

Positiv

+ + + + Deutschland + + + Müller, Hansi + + + + + + Custom ID 105 + Testtitel 5 + 1. Januar 1950 - 31. Dezember 1953 + Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo + ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur + ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa + quis enim. Donec. + Ankauf + + Foto + 1 Bild + + + +

+ + + + Technische Daten / Farbigkeit +

s/w

+ + + Technische Daten / Fototyp +

Positiv

+ + + + Deutschland + + + Müller, Hansi + + + + + + + + diff --git a/Kitodo/src/test/resources/rulesets/ruleset_ead.xml b/Kitodo/src/test/resources/rulesets/ruleset_ead.xml new file mode 100755 index 00000000000..8adbe1ed721 --- /dev/null +++ b/Kitodo/src/test/resources/rulesets/ruleset_ead.xml @@ -0,0 +1,477 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Kitodo/src/test/resources/xslt/ead2kitodo.xsl b/Kitodo/src/test/resources/xslt/ead2kitodo.xsl new file mode 100644 index 00000000000..519f1770dfe --- /dev/null +++ b/Kitodo/src/test/resources/xslt/ead2kitodo.xsl @@ -0,0 +1,255 @@ + + + + + + + + + + + + + + + + + verzeichnungseinheit + + + + + + + + + vorgang + + + + + + + + + UNKNOWN + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Kitodo/src/test/resources/xslt/eadParent2kitodo.xsl b/Kitodo/src/test/resources/xslt/eadParent2kitodo.xsl new file mode 100644 index 00000000000..ef09fc7b146 --- /dev/null +++ b/Kitodo/src/test/resources/xslt/eadParent2kitodo.xsl @@ -0,0 +1,115 @@ + + + + + + + + + + + + + + + + + bestand + + + + + + + + + klassifikation + + + + + + + + + series + + + + + + + + + UNKNOWN + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 2d691c58fd1dff1d9447d10a2369d2b594791dbc Mon Sep 17 00:00:00 2001 From: Arved Solth Date: Tue, 12 Nov 2024 12:32:04 +0100 Subject: [PATCH 2/5] Implement change requests regarding JavaDoc --- .../createprocess/CatalogImportDialog.java | 2 +- .../forms/createprocess/CreateProcessForm.java | 6 ++++++ .../kitodo/production/model/LazyHitModel.java | 4 +++- .../production/services/data/ImportService.java | 17 +++++++++++++++++ .../thread/ImportEadProcessesThread.java | 2 +- 5 files changed, 28 insertions(+), 3 deletions(-) diff --git a/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/CatalogImportDialog.java b/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/CatalogImportDialog.java index c6cabb9e6c6..18af81e1cc1 100644 --- a/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/CatalogImportDialog.java +++ b/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/CatalogImportDialog.java @@ -428,7 +428,7 @@ public String getSelectedField() { /** * Set selectedField. * - * @param field as java.lang.String + * @param field as String */ public void setSelectedField(String field) { this.selectedField = field; diff --git a/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/CreateProcessForm.java b/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/CreateProcessForm.java index 0eaf73f9daa..fcc74ccf73a 100644 --- a/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/CreateProcessForm.java +++ b/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/CreateProcessForm.java @@ -122,6 +122,12 @@ public CreateProcessForm() { this.priorityList = priorityList; } + /** + * Calculate number of EAD elements of selected level (e.g. "item", "file" etc.) from "xmlString", containing + * content of currently imported XML file. + * + * @throws XMLStreamException when retrieving EAD from XML data fails + */ public void calculateNumberOfEadElements() throws XMLStreamException { numberOfEadElements = XMLUtils.getNumberOfElements(xmlString, selectedEadLevel); } diff --git a/Kitodo/src/main/java/org/kitodo/production/model/LazyHitModel.java b/Kitodo/src/main/java/org/kitodo/production/model/LazyHitModel.java index 0ec0878e86c..2bc61a38bcf 100644 --- a/Kitodo/src/main/java/org/kitodo/production/model/LazyHitModel.java +++ b/Kitodo/src/main/java/org/kitodo/production/model/LazyHitModel.java @@ -30,7 +30,9 @@ public class LazyHitModel extends LazyDataModel { private SearchResult searchResult = null; /** - * Empty default constructor. Sets default catalog and search field, if configured. + * Constructor setting this LazyHitModels 'CatalogImportDialog'. + * + * @param dialog as CatalogImportDialog */ public LazyHitModel(CatalogImportDialog dialog) { this.catalogImportDialog = dialog; diff --git a/Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java b/Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java index 44a2704f91a..2eb73aec322 100644 --- a/Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java +++ b/Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java @@ -405,10 +405,14 @@ public String getParentID(Document document, String higherLevelIdentifier, Strin /** * Creates a temporary Process from the given document with templateID und projectID. + * @param importConfiguration ImportConfiguration used to create TempProcess * @param document the given document * @param templateID the template to use * @param projectID the project to use * @return a temporary process + * @throws ProcessGenerationException when creating process for given template and project fails + * @throws IOException when loading workpiece of TempProcess or retrieving type of document fails + * @throws TransformerException when loading workpiece of TempProcess fails */ public TempProcess createTempProcessFromDocument(ImportConfiguration importConfiguration, Document document, int templateID, int projectID) @@ -612,6 +616,10 @@ private void importParents(String recordId, ImportConfiguration importConfigurat /** * Check if there already is a parent process in Database. + * + * @param parentID ID of parent process to retrieve + * @param ruleset ruleset of parent process to retrieve + * @param projectID ID of project to which parent process must belong */ public void checkForParent(String parentID, Ruleset ruleset, int projectID) { this.parentTempProcess = retrieveParentTempProcess(parentID, ruleset, projectID); @@ -835,6 +843,15 @@ private List getEADElements(Document document, String level) { StringConstants.LEVEL, level); } + /** + * Check and return whether maximum number of tags with given tag name 'tagName' in XML with content given as + * 'xmlString' exceeds limit configured as maxNumberOfProcessesForImportMask in kitodo_config.properties. + * + * @param xmlString String representation of XML content to check + * @param tagName name of XML tag counted in XML content + * @return whether the number of tags exceeds the allowed limit + * @throws XMLStreamException when retrieving number of tags in XML content fails + */ public boolean isMaxNumberOfRecordsExceeded(String xmlString, String tagName) throws XMLStreamException { int numberOfRecords = XMLUtils.getNumberOfElements(xmlString, tagName); return numberOfRecords > ConfigCore.getIntParameterOrDefaultValue(ParameterCore.MAX_NUMBER_OF_PROCESSES_FOR_IMPORT_MASK); diff --git a/Kitodo/src/main/java/org/kitodo/production/thread/ImportEadProcessesThread.java b/Kitodo/src/main/java/org/kitodo/production/thread/ImportEadProcessesThread.java index 0b8ec0abe88..46247c9d775 100644 --- a/Kitodo/src/main/java/org/kitodo/production/thread/ImportEadProcessesThread.java +++ b/Kitodo/src/main/java/org/kitodo/production/thread/ImportEadProcessesThread.java @@ -230,7 +230,7 @@ public void run() { } private void cleanUpProcesses(List processIds, int newParentId) { - logger.error("Deleting processes created until this point to resolve erroneous intermediate state."); + logger.info("Deleting processes created until this point to resolve erroneous intermediate state."); // cleanup any processes that might have been created until this point for (int id : processIds) { try { From 685c12bc85616ef36c06bf4f3de17967d0e542d3 Mon Sep 17 00:00:00 2001 From: Arved Solth Date: Thu, 14 Nov 2024 14:45:34 +0100 Subject: [PATCH 3/5] Translate error message --- .../org/kitodo/production/services/data/ImportService.java | 5 ++--- Kitodo/src/main/resources/messages/errors_de.properties | 1 + Kitodo/src/main/resources/messages/errors_en.properties | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java b/Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java index 2eb73aec322..f0ef984f1eb 100644 --- a/Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java +++ b/Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java @@ -786,9 +786,8 @@ public LinkedList parseImportedEADCollection(DataRecord importedEAD List childElements = getEADElements(eadCollectionDocument, eadChildProcessLevel); if (parentElements.size() != 1) { - throw new ProcessGenerationException( - String.format("EAD XML does not contain exactly one element of parent level '%s'!", - eadParentProcessLevel)); + throw new ProcessGenerationException(Helper.getTranslation( + "importError.wrongNumberOfEadParentLevelElements", eadParentProcessLevel)); } // create temp processes for parent (e.g. "collection") and children (e.g. "files") diff --git a/Kitodo/src/main/resources/messages/errors_de.properties b/Kitodo/src/main/resources/messages/errors_de.properties index 5204a55a81b..dcd4eaed90e 100644 --- a/Kitodo/src/main/resources/messages/errors_de.properties +++ b/Kitodo/src/main/resources/messages/errors_de.properties @@ -83,6 +83,7 @@ errorImporting=Fehler beim Importieren von PPN {0}: {1} importFailedError=Der Import von {0} ist fehlgeschlagen. Die Fehlermeldung lautet\: {1}. imagePaginationError=Es wurden {1} Bilder gefunden, aber {0} Bilder erwartet. Bitte \u00FCberpr\u00FCfen Sie die Paginierung. importError.emptyDocument=Vorgang konnte nicht aus geladenem XML-Dokument erzeugt werden. +importError.wrongNumberOfEadParentLevelElements=EAD-XML enth\u00E4lt nicht genau ein Element vom ausgew\u00E4hltem EAD-Level "{0}"! invalidIdentifierCharacter=Der Wert des Identifikationsmerkmals {0} enth\u00E4lt ung\u00FCltige Zeichen. invalidIdentifierSame=Das Identifikationsmerkmal {0} enth\u00E4lt in {1} und {2} den gleichen Wert. diff --git a/Kitodo/src/main/resources/messages/errors_en.properties b/Kitodo/src/main/resources/messages/errors_en.properties index 1f56eeebc3d..d6cce6df241 100644 --- a/Kitodo/src/main/resources/messages/errors_en.properties +++ b/Kitodo/src/main/resources/messages/errors_en.properties @@ -83,6 +83,7 @@ errorImporting=Error importing PPN {0}: {1} importFailedError=The import of {0} failed. The error message is\: {1} imagePaginationError={1} images found but {0} images expected. Please check the pagination. importError.emptyDocument=Unable to create process from imported XML document. +importError.wrongNumberOfEadParentLevelElements=EAD XML does not contain exactly one element of parent level "{0}"! invalidIdentifierCharacter=The value of the identifier {0} contains invalid characters. invalidIdentifierSame=The identifier {0} has the same value in {1} and {2}. From 41c5e2134e572dbdac86a94c5aead08167334048 Mon Sep 17 00:00:00 2001 From: Arved Solth Date: Thu, 14 Nov 2024 14:47:15 +0100 Subject: [PATCH 4/5] Rename method 'getNumberOfElements'. --- .../production/forms/createprocess/CreateProcessForm.java | 2 +- Kitodo/src/main/java/org/kitodo/production/helper/XMLUtils.java | 2 +- .../java/org/kitodo/production/services/data/ImportService.java | 2 +- .../org/kitodo/production/thread/ImportEadProcessesThread.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/CreateProcessForm.java b/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/CreateProcessForm.java index fcc74ccf73a..a9f6d425b8e 100644 --- a/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/CreateProcessForm.java +++ b/Kitodo/src/main/java/org/kitodo/production/forms/createprocess/CreateProcessForm.java @@ -129,7 +129,7 @@ public CreateProcessForm() { * @throws XMLStreamException when retrieving EAD from XML data fails */ public void calculateNumberOfEadElements() throws XMLStreamException { - numberOfEadElements = XMLUtils.getNumberOfElements(xmlString, selectedEadLevel); + numberOfEadElements = XMLUtils.getNumberOfEADElements(xmlString, selectedEadLevel); } /** diff --git a/Kitodo/src/main/java/org/kitodo/production/helper/XMLUtils.java b/Kitodo/src/main/java/org/kitodo/production/helper/XMLUtils.java index 8cb21648749..d208e92f954 100644 --- a/Kitodo/src/main/java/org/kitodo/production/helper/XMLUtils.java +++ b/Kitodo/src/main/java/org/kitodo/production/helper/XMLUtils.java @@ -272,7 +272,7 @@ public static DataRecord createRecordFromXMLElement(String xmlContent, ImportCon * @return number of EAD elements with given level * @throws XMLStreamException when parsing XML string fails */ - public static int getNumberOfElements(String xmlString, String eadLevel) throws XMLStreamException { + public static int getNumberOfEADElements(String xmlString, String eadLevel) throws XMLStreamException { int count = 0; XMLInputFactory factory = XMLInputFactory.newInstance(); XMLStreamReader reader = factory.createXMLStreamReader(new StringReader(xmlString)); diff --git a/Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java b/Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java index f0ef984f1eb..7c83cc1d995 100644 --- a/Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java +++ b/Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java @@ -852,7 +852,7 @@ private List getEADElements(Document document, String level) { * @throws XMLStreamException when retrieving number of tags in XML content fails */ public boolean isMaxNumberOfRecordsExceeded(String xmlString, String tagName) throws XMLStreamException { - int numberOfRecords = XMLUtils.getNumberOfElements(xmlString, tagName); + int numberOfRecords = XMLUtils.getNumberOfEADElements(xmlString, tagName); return numberOfRecords > ConfigCore.getIntParameterOrDefaultValue(ParameterCore.MAX_NUMBER_OF_PROCESSES_FOR_IMPORT_MASK); } diff --git a/Kitodo/src/main/java/org/kitodo/production/thread/ImportEadProcessesThread.java b/Kitodo/src/main/java/org/kitodo/production/thread/ImportEadProcessesThread.java index 46247c9d775..465e113752f 100644 --- a/Kitodo/src/main/java/org/kitodo/production/thread/ImportEadProcessesThread.java +++ b/Kitodo/src/main/java/org/kitodo/production/thread/ImportEadProcessesThread.java @@ -127,7 +127,7 @@ public void run() { List newProcessIds = new ArrayList<>(); int newParentId = 0; try { - int numberOfElements = XMLUtils.getNumberOfElements(xmlString, eadLevel); + int numberOfElements = XMLUtils.getNumberOfEADElements(xmlString, eadLevel); XMLInputFactory inputFactory = XMLInputFactory.newInstance(); XMLEventReader eventReader = inputFactory.createXMLEventReader(new StringReader(xmlString)); boolean inProcessElement = false; From 75dd227d6cfd7b02c71de2625f41f75c4c4843e7 Mon Sep 17 00:00:00 2001 From: Arved Solth Date: Fri, 15 Nov 2024 12:38:38 +0100 Subject: [PATCH 5/5] Add parameter to control execption handling during EAD collection import --- .../java/org/kitodo/config/enums/ParameterCore.java | 8 +++++++- .../production/thread/ImportEadProcessesThread.java | 12 +++++++++++- Kitodo/src/main/resources/kitodo_config.properties | 7 +++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/Kitodo/src/main/java/org/kitodo/config/enums/ParameterCore.java b/Kitodo/src/main/java/org/kitodo/config/enums/ParameterCore.java index 3aa64a37153..8c41d1540e2 100644 --- a/Kitodo/src/main/java/org/kitodo/config/enums/ParameterCore.java +++ b/Kitodo/src/main/java/org/kitodo/config/enums/ParameterCore.java @@ -661,7 +661,13 @@ public enum ParameterCore implements ParameterInterface { * Optional parameter controlling how many processes are to be displayed and processed in the metadata import mask. * When more data records are imported the import process is moved to a background task. Default value is 5. */ - MAX_NUMBER_OF_PROCESSES_FOR_IMPORT_MASK(new Parameter<>("maxNumberOfProcessesForImportMask", 5)); + MAX_NUMBER_OF_PROCESSES_FOR_IMPORT_MASK(new Parameter<>("maxNumberOfProcessesForImportMask", 5)), + + /* + * Optional parameter controlling whether the import of all elements from an uploaded EAD XML file should be + * canceled when an exception occurs or not. Defaults to 'false'. + */ + STOP_EAD_COLLECTION_IMPORT_ON_EXCEPTION(new Parameter<>("stopEadCollectionImportOnException", false)); private final Parameter parameter; diff --git a/Kitodo/src/main/java/org/kitodo/production/thread/ImportEadProcessesThread.java b/Kitodo/src/main/java/org/kitodo/production/thread/ImportEadProcessesThread.java index 465e113752f..a9cd0c309f5 100644 --- a/Kitodo/src/main/java/org/kitodo/production/thread/ImportEadProcessesThread.java +++ b/Kitodo/src/main/java/org/kitodo/production/thread/ImportEadProcessesThread.java @@ -45,6 +45,8 @@ import org.kitodo.api.MetadataEntry; import org.kitodo.api.dataeditor.rulesetmanagement.FunctionalMetadata; import org.kitodo.api.dataeditor.rulesetmanagement.RulesetManagementInterface; +import org.kitodo.config.ConfigCore; +import org.kitodo.config.enums.ParameterCore; import org.kitodo.constants.StringConstants; import org.kitodo.data.database.beans.Client; import org.kitodo.data.database.beans.ImportConfiguration; @@ -126,6 +128,7 @@ public void run() { setAuthenticatedUser(); List newProcessIds = new ArrayList<>(); int newParentId = 0; + boolean stopOnError = ConfigCore.getBooleanParameter(ParameterCore.STOP_EAD_COLLECTION_IMPORT_ON_EXCEPTION); try { int numberOfElements = XMLUtils.getNumberOfEADElements(xmlString, eadLevel); XMLInputFactory inputFactory = XMLInputFactory.newInstance(); @@ -197,7 +200,14 @@ public void run() { inProcessElement = false; String content = event.toString(); stringBuilder.append(removeDefaultNamespaceUri(content)); - newProcessIds.add(parseXmlStringToProcessedTempProcess(stringBuilder.toString()).getProcess().getId()); + try { + newProcessIds.add(parseXmlStringToProcessedTempProcess(stringBuilder.toString()).getProcess().getId()); + } catch (Exception e) { + logger.error(e.getMessage(), e); + if (stopOnError) { + throw new ProcessGenerationException("Unable to create process. Cause: " + e.getMessage()); + } + } stringBuilder = new StringBuilder(); } else { if (inParentProcessElement || inProcessElement) { diff --git a/Kitodo/src/main/resources/kitodo_config.properties b/Kitodo/src/main/resources/kitodo_config.properties index 99a9a591a82..9fe425ec510 100644 --- a/Kitodo/src/main/resources/kitodo_config.properties +++ b/Kitodo/src/main/resources/kitodo_config.properties @@ -796,3 +796,10 @@ security.secret.ldapManagerPassword= # informing the user that the processes will be imported using a background task and the progress can be tracked via the # task manager. maxNumberOfProcessesForImportMask=5 + +# The parameter 'stopEadCollectionImportOnException' can be used to control how Kitodo should handle potential +# exceptions occurring during the import of EAD collections. When set to 'true', the import of an upload EAD XML file +# will be canceled and all new processes created from the uploaded file up to this point are removed. If set to 'false', +# the import will skip the current EAD element that caused the exception and continue with the next element. +# Defaults to 'false'. +stopEadCollectionImportOnException=false