From 0e0dcf8b8cbcd2bd9be5f241735cd481b3cc3865 Mon Sep 17 00:00:00 2001 From: Matthias Ronge Date: Tue, 20 Feb 2024 16:27:02 +0100 Subject: [PATCH] Adds a way to import processes that only exist as files (#5903) * Create a base for the Kitodo script * Add a button in the Kitodo Script Dialog * Implement * Add tests * Fix and complete messages * Improve readability * Fix checkstyle * Fix checkstyle * Fix checkstyle * Print meaningful exception * Print meaningful exception manually * Fix a control flow error * Remove error wrapper * Test more generically, and give better error message on failure * Fixes a typo * Fix progress not working * Prevents tasks not being indexed * Support test file arrangement before and after PR #5876 * Fix Javadoc * Remove 'final' modifier from functions used in constructors Since private methods cannot be meaningfully overridden because of their visibility, declaring them final is redundant. * Use Files.walk() with try-with-resources * Invert if-condition * Use Files.walk() with try-with-resource * Invert if-condition * Fix Javadoc * Fix a typo * Spell out abbreviated negations in English * Remove duplicate class 'TreeDeleter' * Fix indent * Reformat try-with-resources statements, remove unnecessary ';' * Rewrite childDepth() to make use of OptionalInt * Fix checkstyle * Rearrange function calls and checks The order of execution of the underlying tasks depends on the order in which they are found on the disk and is unpredictable. Reliable testing is only possible after the end of this group of tasks. * Remove support for test files before PR #5876 * Use 'ProcessTestUtils' for clean-up * Remove database IDs from test --- .../metadata/MetadataValidationInterface.java | 4 +- .../metadata/MetadataValidation.java | 10 +- .../org/kitodo/production/helper/Helper.java | 3 + .../services/command/ImportProcesses.java | 347 ++++++++++ .../services/command/ImportingProcess.java | 610 ++++++++++++++++++ .../services/command/KitodoScriptService.java | 12 + .../validation/MetadataValidationService.java | 32 +- .../resources/messages/errors_de.properties | 16 + .../resources/messages/errors_en.properties | 16 + .../resources/messages/errors_es.properties | 16 + .../resources/messages/messages_de.properties | 1 + .../resources/messages/messages_en.properties | 1 + .../resources/messages/messages_es.properties | 1 + .../executeScriptSelectedPopup.xhtml | 3 + .../services/command/ImportProcessesIT.java | 332 ++++++++++ .../ImportProcessesIT/p1_valid/meta.xml | 20 + .../images/17_123_0001_media/00000001.jpg | Bin 0 -> 1155 bytes .../ImportProcessesIT/p1c1_valid/meta.xml | 12 + .../images/17_123_0002_media/00000001.jpg | Bin 0 -> 1155 bytes .../ImportProcessesIT/p1c2_valid/meta.xml | 12 + .../p2_parentMissingAChild/meta.xml | 20 + .../ImportProcessesIT/p2c1_valid/meta.xml | 12 + .../ImportProcessesIT/p3_not-valid/meta.xml | 14 + .../ImportProcessesIT/p3c1_valid/meta.xml | 12 + .../p4_valid_butChildIsNot/meta.xml | 14 + .../ImportProcessesIT/p4c1_not-valid/meta.xml | 11 + .../resources/rulesets/ImportProcessesIT.xml | 100 +++ 27 files changed, 1620 insertions(+), 11 deletions(-) create mode 100644 Kitodo/src/main/java/org/kitodo/production/services/command/ImportProcesses.java create mode 100644 Kitodo/src/main/java/org/kitodo/production/services/command/ImportingProcess.java create mode 100644 Kitodo/src/test/java/org/kitodo/production/services/command/ImportProcessesIT.java create mode 100644 Kitodo/src/test/resources/ImportProcessesIT/p1_valid/meta.xml create mode 100644 Kitodo/src/test/resources/ImportProcessesIT/p1c1_valid/images/17_123_0001_media/00000001.jpg create mode 100644 Kitodo/src/test/resources/ImportProcessesIT/p1c1_valid/meta.xml create mode 100644 Kitodo/src/test/resources/ImportProcessesIT/p1c2_valid/images/17_123_0002_media/00000001.jpg create mode 100644 Kitodo/src/test/resources/ImportProcessesIT/p1c2_valid/meta.xml create mode 100644 Kitodo/src/test/resources/ImportProcessesIT/p2_parentMissingAChild/meta.xml create mode 100644 Kitodo/src/test/resources/ImportProcessesIT/p2c1_valid/meta.xml create mode 100644 Kitodo/src/test/resources/ImportProcessesIT/p3_not-valid/meta.xml create mode 100644 Kitodo/src/test/resources/ImportProcessesIT/p3c1_valid/meta.xml create mode 100644 Kitodo/src/test/resources/ImportProcessesIT/p4_valid_butChildIsNot/meta.xml create mode 100644 Kitodo/src/test/resources/ImportProcessesIT/p4c1_not-valid/meta.xml create mode 100644 Kitodo/src/test/resources/rulesets/ImportProcessesIT.xml diff --git a/Kitodo-API/src/main/java/org/kitodo/api/validation/metadata/MetadataValidationInterface.java b/Kitodo-API/src/main/java/org/kitodo/api/validation/metadata/MetadataValidationInterface.java index 375fffb7e38..10b927bfcf1 100644 --- a/Kitodo-API/src/main/java/org/kitodo/api/validation/metadata/MetadataValidationInterface.java +++ b/Kitodo-API/src/main/java/org/kitodo/api/validation/metadata/MetadataValidationInterface.java @@ -67,8 +67,10 @@ ValidationResult validate(URI metsFileUri, URI rulesetFileUri, List metadataLanguage, Map translations); + List metadataLanguage, Map translations, boolean checkMedia); } diff --git a/Kitodo-Validation/src/main/java/org/kitodo/validation/metadata/MetadataValidation.java b/Kitodo-Validation/src/main/java/org/kitodo/validation/metadata/MetadataValidation.java index 5001e8f233c..179dd4edb6d 100644 --- a/Kitodo-Validation/src/main/java/org/kitodo/validation/metadata/MetadataValidation.java +++ b/Kitodo-Validation/src/main/java/org/kitodo/validation/metadata/MetadataValidation.java @@ -106,7 +106,7 @@ public ValidationResult validate(URI metsFileUri, URI rulesetFileUri, List metadataLanguage, Map translations) { + List metadataLanguage, Map translations, boolean checkMedia) { Collection results = new ArrayList<>(); - results.add(checkForStructuresWithoutMedia(workpiece, translations)); - results.add(checkForUnlinkedMedia(workpiece, translations)); + if (checkMedia) { + results.add(checkForStructuresWithoutMedia(workpiece, translations)); + results.add(checkForUnlinkedMedia(workpiece, translations)); + } for (LogicalDivision logicalDivision : workpiece.getAllLogicalDivisions()) { results.addAll(checkMetadataRules(logicalDivision.toString(), logicalDivision.getType(), diff --git a/Kitodo/src/main/java/org/kitodo/production/helper/Helper.java b/Kitodo/src/main/java/org/kitodo/production/helper/Helper.java index 955cd7c66f3..92e90b6eb14 100644 --- a/Kitodo/src/main/java/org/kitodo/production/helper/Helper.java +++ b/Kitodo/src/main/java/org/kitodo/production/helper/Helper.java @@ -489,6 +489,9 @@ public static String getTranslation(String title, String... insertions) { private static String appendUnusedInsertions(String message, String... insertions) { StringBuilder messageBuilder = new StringBuilder(message); for (String insertion : insertions) { + if (Objects.isNull(insertion)) { + continue; + } String separator = ": "; insertion = Objects.toString(insertion); if (!messageBuilder.toString().contains(insertion)) { diff --git a/Kitodo/src/main/java/org/kitodo/production/services/command/ImportProcesses.java b/Kitodo/src/main/java/org/kitodo/production/services/command/ImportProcesses.java new file mode 100644 index 00000000000..ddd4dd19605 --- /dev/null +++ b/Kitodo/src/main/java/org/kitodo/production/services/command/ImportProcesses.java @@ -0,0 +1,347 @@ +/* + * (c) Kitodo. Key to digital objects e. V. + * + * This file is part of the Kitodo project. + * + * It is licensed under GNU General Public License version 3 or later. + * + * For the full copyright and license information, please read the + * GPL3-License.txt file that was distributed with this source code. + */ + +package org.kitodo.production.services.command; + +// base Java +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Comparator; +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +// open source code +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.kitodo.api.dataeditor.rulesetmanagement.RulesetManagementInterface; +import org.kitodo.config.ConfigCore; +import org.kitodo.config.KitodoConfig; +import org.kitodo.config.enums.ParameterCore; +import org.kitodo.data.database.beans.Project; +import org.kitodo.data.database.beans.Template; +import org.kitodo.data.database.exceptions.DAOException; +import org.kitodo.data.exceptions.DataException; +import org.kitodo.exceptions.InvalidImagesException; +import org.kitodo.exceptions.MediaNotFoundException; +import org.kitodo.exceptions.ProcessGenerationException; +import org.kitodo.production.helper.Helper; +import org.kitodo.production.helper.tasks.EmptyTask; +import org.kitodo.production.services.ServiceManager; +import org.kitodo.production.services.data.ProjectService; +import org.kitodo.production.services.data.TemplateService; + + +/** + * Long-running task importing processes into Production. + */ +public final class ImportProcesses extends EmptyTask { + private static final Logger logger = LogManager.getLogger(ImportProcesses.class); + + // number of actions required for initialization + private static final int INIT_ACTIONS_COUNT = 1; + // number of actions required for validation + private static final int VALIDATION_ACTIONS_COUNT = 1; + + // services required + private final ProjectService projectService = ServiceManager.getProjectService(); + private final TemplateService templateService = ServiceManager.getTemplateService(); + + // global system configuration + private final boolean strictValidation = ConfigCore.getBooleanParameter(ParameterCore.VALIDATION_FAIL_ON_WARNING); + + // data + private final Path importRootPath; + private final Project project; + private final Template templateForProcesses; + private RulesetManagementInterface ruleset; + private final Path errorPath; + private final TreeMap importingProcesses; + private final int numberOfImportingProcesses; + + // thread execution + private int step = 0; + int totalActions = 1; + private Iterator importingProcessesIterator; + ImportingProcess validatingImportingProcess; + private ImportingProcess currentlyImporting; + private int nextAction = 0; + private int numberOfRemainingActions = 0; + + + /** + * Constructor. Creates a {@code ProcessesImport} + * long-running task. + * + * @param indir + * input by the user for the source root directory. Can + * [invalidly] be {@code null} if the user has not specified it. + * @param project + * input by the user for the project ID. Can [invalidly] be + * {@code null} if the user has not specified it. + * @param template + * input by the for the process template ID. Can [invalidly] be + * {@code null} if the user has not specified it. + * @param errors + * input by the user for the source root directory. Can [validly] + * be {@code null} if the user has not specified it. + * @throws IllegalArgumentException + * if the user hasn't specified all the necessary parameters, or + * if the parameters have unusable values. Exception message is + * a message key that must be resolved via message properties. + */ + public ImportProcesses(String indir, String project, String template, String errors) throws IOException { + super(indir); + this.importRootPath = checkIndir(indir); + this.project = checkProject(project); + this.templateForProcesses = checkTemplate(template); + this.errorPath = checkErrors(errors); + try (Stream pathStream = Files.walk(this.importRootPath, 1)) { + this.importingProcesses = pathStream.filter(Files::isDirectory) + .filter(Predicate.not(this.importRootPath::equals)) + .collect(Collectors.toMap(path -> path.getFileName().toString(), ImportingProcess::new, + (existing, replacing) -> replacing, TreeMap::new)); + } + this.numberOfImportingProcesses = importingProcesses.size(); + } + + /** + * Clone constructor. Creates a copy of the object. But + * because the object is a terminated thread, it makes a new thread that can + * be started again. + */ + private ImportProcesses(ImportProcesses source) { + super(source); + this.importRootPath = source.importRootPath; + this.project = source.project; + this.templateForProcesses = source.templateForProcesses; + this.ruleset = source.ruleset; + this.errorPath = source.errorPath; + this.importingProcesses = source.importingProcesses; + this.numberOfImportingProcesses = source.numberOfImportingProcesses; + + this.step = source.step; + this.totalActions = source.totalActions; + this.importingProcessesIterator = source.importingProcessesIterator; + this.validatingImportingProcess = source.validatingImportingProcess; + this.currentlyImporting = source.currentlyImporting; + this.nextAction = source.nextAction; + this.numberOfRemainingActions = source.numberOfRemainingActions; + } + + /** + * Checks whether the {@code indir} parameter is specified and valid. If + * not, a corresponding error message is thrown as an exception. (This is + * picked up above and will be translated and displayed to the user.) The + * {@code indir} directory must be specified, it must exist, and the Tomcat + * must have permission to get the directory listing (execution permission). + * + * @param indir + * directory entered by the user. Can be {@code null} if the + * parameter is not specified + * @return a Path object to the root directory for bulk import + */ + private Path checkIndir(String indir) { + if (Objects.isNull(indir)) { + throw new IllegalArgumentException("kitodoScript.importProcesses.indir.isNull"); + } + Path importRoot = Paths.get(indir); + if (!Files.isDirectory(importRoot)) { + throw new IllegalArgumentException("kitodoScript.importProcesses.indir.isNoDirectory"); + } + if (!Files.isExecutable(importRoot)) { + throw new IllegalArgumentException("kitodoScript.importProcesses.indir.cannotExecute"); + } + return importRoot; + } + + /** + * Checks whether the parameter {@code project} is specified and valid. If + * not, a corresponding error message is thrown as an exception. (This is + * picked up above and will be translated and displayed to the user.) The + * project must be specified, it must be syntactically valid (a positive + * integer), and a project with that ID must exist. + * + * @param project + * user-entered project number. Can be {@code null} if the + * parameter is not specified + * @return the project object from the database + */ + private Project checkProject(String project) { + if (Objects.isNull(project)) { + throw new IllegalArgumentException("kitodoScript.importProcesses.project.isNull"); + } + if (!project.matches("[\\d]+")) { + throw new IllegalArgumentException("kitodoScript.importProcesses.project.isNoProjectID"); + } + Integer projectInteger = Integer.valueOf(project); + try { + return projectService.getById(projectInteger); + } catch (DAOException e) { + logger.catching(e); + throw new IllegalArgumentException("kitodoScript.importProcesses.project.noProjectWithID"); + } + } + + /** + * Checks whether the parameter {@code template} is specified and valid. If + * not, a corresponding error message is thrown as an exception. (This is + * picked up above and will be translated and displayed to the user.) The + * production template must be specified, its ID must be syntactically valid + * (a positive integer), and a production template with that ID must exist. + * + * @param template + * user-entered production template number. Can be {@code null} + * if the parameter is not specified + * @return the production template object from the database + */ + private Template checkTemplate(String template) { + if (Objects.isNull(template)) { + throw new IllegalArgumentException("kitodoScript.importProcesses.template.isNull"); + } + if (!template.matches("[\\d]+")) { + throw new IllegalArgumentException("kitodoScript.importProcesses.template.isNoTemplateID"); + } + Integer templateInteger = Integer.valueOf(template); + try { + return templateService.getById(templateInteger); + } catch (DAOException e) { + logger.catching(e); + throw new IllegalArgumentException("kitodoScript.importProcesses.template.noTemplateWithID"); + } + } + + /** + * Checks whether the parameter {@code errors} is specified and—if so—is + * valid. If not, a corresponding error message is thrown as an exception. + * (This is picked up above and will be translated and displayed to the + * user.) The errors directory is optional, but if provided, it must exist + * and be writable by Tomcat. + * + * @param errors + * Path to directory for errors. May be {@code null} if not + * specified + * @return a Path object to the errors directory, or {@code null} if not + * specified + */ + private Path checkErrors(String errors) { + if (Objects.isNull(errors)) { + return null; + } + Path errorDir = Paths.get(errors); + if (!Files.isDirectory(errorDir)) { + throw new IllegalArgumentException("kitodoScript.importProcesses.errors.isNoDirectory"); + } + if (!Files.isWritable(errorDir)) { + throw new IllegalArgumentException("kitodoScript.importProcesses.errors.cannotWrite"); + } + return errorDir; + } + + /* + * The method is used by the Task Manager to spawn a new thread for this + * task, if the previous thread was prematurely stopped. (A thread cannot + * be continued under Java, a new thread object must be required.) + */ + @Override + public ImportProcesses replace() { + return new ImportProcesses(this); + } + + /* + * The thread's runner method. This does the real work. Each importing + * processes are always driven to individual work steps again and again, + * after which the method returns to increase the progress and allow the + * task to be stopped. This is complicated, but gives the task additional + * flexibility to control. + */ + @Override + public void run() { + try { + while (step < totalActions) { + run(step); + super.setProgress(100d * ++step / totalActions); + if (super.isInterrupted()) { + return; + } + } + // error barrier + } catch (IOException | DAOException | DataException | ProcessGenerationException | MediaNotFoundException + | InvalidImagesException | RuntimeException exception) { + Helper.setErrorMessage(exception.getLocalizedMessage(), logger, exception); + super.setException(exception); + } + } + + void run(int setStep) throws IOException, DAOException, DataException, ProcessGenerationException, + MediaNotFoundException, InvalidImagesException { + + step = setStep; + Path processesPath = Paths.get(KitodoConfig.getKitodoDataDirectory()); + if (step == 0) { + initialize(); + } else if (step <= numberOfImportingProcesses) { + validate(); + } else { + copyFilesAndCreateDatabaseEntry(step, processesPath); + } + } + + private void initialize() throws IOException { + super.setWorkDetail(importRootPath.toString()); + ruleset = ServiceManager.getRulesetManagementService().getRulesetManagement(); + ruleset.load(new File(Paths.get(ConfigCore.getParameter(ParameterCore.DIR_RULESETS), + templateForProcesses.getRuleset().getFile()).toString())); + totalActions = importingProcesses.entrySet().parallelStream().map(Entry::getValue) + .mapToInt(ImportingProcess::numberOfActions).sum() + INIT_ACTIONS_COUNT; + importingProcessesIterator = importingProcesses.values().iterator(); + } + + private void validate() throws IOException, DAOException { + validatingImportingProcess = importingProcessesIterator.next(); + super.setWorkDetail(validatingImportingProcess.directoryName); + validatingImportingProcess.validate(ruleset, strictValidation, importingProcesses); + // in last iteration, re-initialize iterator + if (step == numberOfImportingProcesses) { + /* Children must be imported before their parents, so + * that when the parents are imported, the process ID of + * the children is already known and can be written down + * in the METS file. */ + TreeSet childrenFirst = new TreeSet( + Comparator.comparingInt(ImportingProcess::childDepth).thenComparing(ImportingProcess::toString)); + childrenFirst.addAll(importingProcesses.values()); + importingProcessesIterator = childrenFirst.iterator(); + } + } + + private void copyFilesAndCreateDatabaseEntry(int step, Path processesPath) throws IOException, DAOException, DataException, + ProcessGenerationException, MediaNotFoundException, InvalidImagesException { + if (nextAction == numberOfRemainingActions && step < totalActions - 1) { + currentlyImporting = importingProcessesIterator.next(); + currentlyImporting.setProject(project); + currentlyImporting.setTemplate(templateForProcesses); + nextAction = 0; + currentlyImporting.setOutputRoot(currentlyImporting.isCorrect() ? processesPath : errorPath); + numberOfRemainingActions = currentlyImporting.numberOfActions() - VALIDATION_ACTIONS_COUNT; + } + super.setWorkDetail(currentlyImporting.directoryName); + currentlyImporting.executeAction(nextAction); + nextAction++; + } +} diff --git a/Kitodo/src/main/java/org/kitodo/production/services/command/ImportingProcess.java b/Kitodo/src/main/java/org/kitodo/production/services/command/ImportingProcess.java new file mode 100644 index 00000000000..12164defda5 --- /dev/null +++ b/Kitodo/src/main/java/org/kitodo/production/services/command/ImportingProcess.java @@ -0,0 +1,610 @@ +/* + * (c) Kitodo. Key to digital objects e. V. + * + * This file is part of the Kitodo project. + * + * It is licensed under GNU General Public License version 3 or later. + * + * For the full copyright and license information, please read the + * GPL3-License.txt file that was distributed with this source code. + */ + +package org.kitodo.production.services.command; + +// static functions used +import static java.lang.System.lineSeparator; +import static java.util.function.Predicate.not; +import static java.util.stream.Collectors.toList; + +// base Java +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +// open source code +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.kitodo.api.Metadata; +import org.kitodo.api.MetadataEntry; +import org.kitodo.api.dataeditor.rulesetmanagement.RulesetManagementInterface; +import org.kitodo.api.dataformat.LogicalDivision; +import org.kitodo.api.dataformat.Workpiece; +import org.kitodo.api.dataformat.mets.LinkedMetsResource; +import org.kitodo.api.validation.State; +import org.kitodo.api.validation.ValidationResult; +import org.kitodo.data.database.beans.Process; +import org.kitodo.data.database.beans.Project; +import org.kitodo.data.database.beans.Task; +import org.kitodo.data.database.beans.Template; +import org.kitodo.data.database.exceptions.DAOException; +import org.kitodo.data.exceptions.DataException; +import org.kitodo.exceptions.InvalidImagesException; +import org.kitodo.exceptions.MediaNotFoundException; +import org.kitodo.exceptions.ProcessGenerationException; +import org.kitodo.production.helper.Helper; +import org.kitodo.production.process.NewspaperProcessesGenerator; +import org.kitodo.production.process.ProcessGenerator; +import org.kitodo.production.services.ServiceManager; +import org.kitodo.production.services.data.ProcessService; +import org.kitodo.production.services.data.TaskService; +import org.kitodo.production.services.dataformat.MetsService; +import org.kitodo.production.services.file.FileService; +import org.kitodo.production.services.validation.MetadataValidationService; + +/** + * A process to import. For each process to be imported (that is, a process + * directory with a {@code meta.xml} file and sub-folders with media files), + * such an object is created, that imports this process into the application + * logic. The existence of the {@code meta.xml} file is checked and its content + * is validated against the specified ruleset. In the event of an error, an + * error information text file is written. Furthermore, a special treatment + * represents the handling of child processes. + */ +final class ImportingProcess { + private static final Logger logger = LogManager.getLogger(ImportingProcess.class); + + // Constants + private static final String META_FILE_NAME = "meta.xml"; + + // Services in use + private final FileService fileService = ServiceManager.getFileService(); + private final MetadataValidationService metadataValidationService = ServiceManager.getMetadataValidationService(); + private final MetsService metsService = ServiceManager.getMetsService(); + private final ProcessService processService = ServiceManager.getProcessService(); + private final TaskService taskService = ServiceManager.getTaskService(); + + /* + * This class makes extensive use of global variables (fields). This is due + * to the fact that the methods of the class do not run from front to back, + * but the sub-steps are called individually by the task manager in order to + * implement the ability to stop and restart the task. + */ + + // Input directories and files + private Path sourceDir; + final String directoryName; + private List filesAndDirectories = new ArrayList<>(); + private int numberOfFileSystemItems; + private Iterator filesAndDirectoriesIterator; + + // Process hierarchy + private Map importingProcesses; + private ImportingProcess parent; + private final List children = new ArrayList<>(); + + // Errors + private Set sharedErroneousProcesses = null; + final List errors = new ArrayList<>(); + + // database process + private Project project; + private Template template; + private String processTitleRule; + private String title; + private String baseType; + private Integer processId; + + // Output directories + private Path copyToRoot; + private Path outputDir; + + /** + * Constructor. Creates a new Importing Process. + * + * @param sourceDir + * source directory of process to import + */ + ImportingProcess(Path sourceDir) { + this.sourceDir = sourceDir; + this.directoryName = sourceDir.getFileName().toString(); + try (Stream pathStream = Files.walk(sourceDir)) { + for (Path entry : (Iterable) pathStream::iterator) { + if (!entry.equals(sourceDir)) { + filesAndDirectories.add(sourceDir.relativize(entry)); + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + numberOfFileSystemItems = filesAndDirectories.size(); + } + + /** + * Specifies the number of actions required to import this process. This is + * used to display the progress bar in Task Manager. + * + *

+ * One action is the validation. One action is to create the directory for + * the import. One action is either creating the process in the database + * or writing the error file. One action each is creating a + * sub-directory or copying a file. + * + * @return sum of actions needed + */ + int numberOfActions() { + int result = numberOfFileSystemItems + 3; + if (logger.isTraceEnabled()) { + logger.trace("Import folder {}: {} actions required. List: {}", this.directoryName, result, + filesAndDirectories.stream().map(Objects::toString).collect(Collectors.joining(", "))); + } + return result; + } + + /** + * Validates the METS file against the ruleset, generates the process title, + * and checks for the existence of all child processes listed. Validation is + * performed extensively to uncover and report as many bugs as possible in + * one run. + * + * @param ruleset + * the ruleset to validate against + * @param strictValidation + * whether warnings are an error + * @param importingProcesses + * access to all processes in this import run, to confirm the + * presence of child processes + */ + void validate(RulesetManagementInterface ruleset, boolean strictValidation, + Map importingProcesses) throws IOException, DAOException { + + this.importingProcesses = importingProcesses; + logger.info("Starting to validate " + this.directoryName); + Path metaFilePath = sourceDir.resolve(META_FILE_NAME); + Workpiece workpiece = metsService.loadWorkpiece(metaFilePath.toUri()); + validateMetsFile(ruleset, strictValidation, workpiece); + validateChildren(workpiece.getLogicalStructure()); + if (!errors.isEmpty()) { + getSharedErroneousProcesses().add(this.directoryName); + } + + // determine data for process object + baseType = workpiece.getLogicalStructure().getType(); + title = formProcessTitle(ruleset, workpiece); + + logger.info("Validation of " + this.directoryName + (errors.isEmpty() ? " completed without errors" + : " completed with errors:" + lineSeparator() + String.join(lineSeparator(), errors))); + } + + /** + * Validates the METS file against the ruleset. Possible rule violations + * will be reported. + * + * @param ruleset + * the ruleset to validate against + * @param strictValidation + * whether warnings are an error + * @param workpiece + * the workpiece to be validated + */ + private void validateMetsFile(RulesetManagementInterface ruleset, boolean strictValidation, Workpiece workpiece) + throws DAOException { + ValidationResult validationResult = metadataValidationService.validate(workpiece, ruleset, false); + State state = validationResult.getState(); + if (State.ERROR.equals(state) || (strictValidation && !State.SUCCESS.equals(state))) { + errors.add(Helper.getTranslation("dataEditor.validation.state.error").concat(":")); + for (String resultMessage : validationResult.getResultMessages()) { + errors.add(" - ".concat(resultMessage)); + } + logger.info(String.join(System.lineSeparator(), errors)); + } + } + + /** + * Checks for the presence of linked child processes. If not, then this will + * break this process, but also any existing child processes. + * + * @param logicalStructure + * in the logical structure of the process, the children are + * listed, if there are any + */ + private void validateChildren(LogicalDivision logicalStructure) { + Set linkedChildren = searchLinkedProcesses(logicalStructure).keySet(); + List problemChildren = linkedChildren.parallelStream().filter(not(importingProcesses::containsKey)) + .collect(toList()); + if (!problemChildren.isEmpty()) { + String errorMessage = Helper.getTranslation("kitodoScript.importProcesses.missingChildren", + this.directoryName, String.join(", ", problemChildren)); + this.errors.add(errorMessage); + } + for (String linkedChild : linkedChildren) { + ImportingProcess importingChild = importingProcesses.get(linkedChild); + if (Objects.nonNull(importingChild)) { + this.children.add(importingChild); + importingChild.parent = this; + } + } + } + + /** + * Recursively determines the existence of linked child processes and + * returns any found ones. + * + * @param division + * division under which the search is made + * @return found references, with name and the division under which it is + * located + */ + private Map searchLinkedProcesses(LogicalDivision division) { + Map result = new HashMap<>(); + if (division.getChildren().isEmpty()) { + if (Objects.nonNull(division.getLink())) { + String name = division.getLink().getUri().toString(); + int firstEqualsSign = name.indexOf('='); + if (firstEqualsSign > -1) { + name = name.substring(firstEqualsSign + 1); + } + result.put(name, division); + } + } else { + for (LogicalDivision child : division.getChildren()) { + result.putAll(searchLinkedProcesses(child)); + } + } + return result; + } + + /** + * Forms the process title. To do this, the formation rule is obtained from + * the ruleset, and then the composition is executed. + * + * @param ruleset + * the ruleset to validate against + * @param workpiece + * the workpiece to be validated + * @return the process title. May be {@code null} if the process title + * cannot yet be formed, because it requires a parent process that + * has not yet been initialized + */ + private String formProcessTitle(RulesetManagementInterface ruleset, Workpiece workpiece) { + Optional processTitleAttribute = ruleset + .getStructuralElementView(baseType, "create", NewspaperProcessesGenerator.ENGLISH).getProcessTitle(); + if (processTitleAttribute.isEmpty()) { + errors.add(Helper.getTranslation("kitodoScript.importProcesses.noProcessTitleRule", baseType)); + return null; + } + processTitleRule = processTitleAttribute.get(); + if (!processTitleRule.startsWith("+") || (parent != null && parent.title != null)) { + String processTitle = calculateProcessTitle(processTitleRule, parent == null ? null : parent.title, + workpiece.getLogicalStructure().getMetadata()); + processTitleRule = null; + return processTitle; + } + return null; + } + + /** + * Calculates the process title. + * + * @param rule + * formation rule + * @param parentTitle + * title of the parent process + * @param metadata + * metadata of the current process + * @return the process title + */ + private String calculateProcessTitle(String rule, String parentTitle, Collection metadata) { + List components = Arrays.asList(rule.split("\\+")); + for (int i = 0; i < components.size(); i++) { + String component = components.get(i); + if (i == 0 && component.isEmpty()) { + components.set(i, parentTitle); + } else if (component.matches("'.*'")) { + components.set(i, component.substring(1, component.length() - 1)); + } else { + Optional lookedUpValue = metadata.parallelStream().filter(MetadataEntry.class::isInstance) + .filter(metadataEntry -> metadataEntry.getKey().equals(component)) + .map(MetadataEntry.class::cast).map(MetadataEntry::getValue).findAny(); + if (lookedUpValue.isPresent()) { + components.set(i, lookedUpValue.get()); + } + } + } + return String.join("", components); + } + + /** + * Returns whether the process to import successfully validated. + * + * @return whether the process is correct + */ + boolean isCorrect() { + getSharedErroneousProcesses(); + if (!errors.isEmpty()) { + sharedErroneousProcesses.add(this.directoryName); + } + return sharedErroneousProcesses.isEmpty(); + } + + private Set getSharedErroneousProcesses() { + if (Objects.isNull(sharedErroneousProcesses)) { + setSharedErroneousProcesses( + Objects.isNull(parent) ? new HashSet<>() : parent.getSharedErroneousProcesses()); + } + return sharedErroneousProcesses; + } + + private void setSharedErroneousProcesses(Set sharedErroneousProcesses) { + this.sharedErroneousProcesses = sharedErroneousProcesses; + for (ImportingProcess importingChild : children) { + importingChild.setSharedErroneousProcesses(sharedErroneousProcesses); + } + } + + // === PROCESSING === + + /** + * Creates the directory where the process should be copied to. It can be + * {@code null} in case of a validation error, then the faulty process is + * not copied anywhere. + * + * @param copyToRoot + * copy target + */ + void setOutputRoot(Path copyToRoot) throws IOException { + this.copyToRoot = copyToRoot; + } + + /** + * Executes a processing step. The method is called from the Task Manager + * once per processing step. Because validation is called separately, the + * number of calls here is one less than the number returned by + * {@link #numberOfActions()}. In other words, {@code action} must be in the + * range 0 to {@code (numberOfActions() - 2)}. + * + * @param action + * processing step + * @throws IOException + * if accessing the file system fails + * @throws DAOException + * if accessing the database fails + * @throws DataException + * if accessing the database fails + * @throws ProcessGenerationException + * if creating the process fails + * @throws InvalidImagesException + * if media file names in the METS file cannot be aligned with + * the project configuration + * @throws MediaNotFoundException + * if media files are missing + */ + void executeAction(int action) throws IOException, DAOException, DataException, ProcessGenerationException, + MediaNotFoundException, InvalidImagesException { + + assert action >= 0 && action <= numberOfFileSystemItems + 1 + : "action out of range: " + action + " [0.." + (numberOfFileSystemItems + 1) + "]"; + if (isCorrect()) { + executeActionForCorrectProcess(action); + } else if (Objects.nonNull(copyToRoot)) { + executeActionForErroreousProcess(action); + } + } + + /** + * Performs a processing step on a successfully validated process. First, + * the process is created in the database to get its ID, which is the name + * of the process directory. In the second step, this directory is created. + * Then all files will be copied into it, except for the {@code meta.xml}, + * which will be skipped. The {@code meta.xml} is transferred at the end and + * adjusted while doing so. + * + * @param action + * processing step + */ + private void executeActionForCorrectProcess(int action) throws IOException, DataException, + ProcessGenerationException, DAOException, InvalidImagesException, MediaNotFoundException { + + if (action == 0) { + if (Objects.nonNull(processTitleRule) && Objects.isNull(title)) { + Workpiece workpiece = metsService.loadWorkpiece(sourceDir.resolve(META_FILE_NAME).toUri()); + title = calculateProcessTitle(processTitleRule, parent.title, + workpiece.getLogicalStructure().getMetadata()); + } + processId = createDatabaseProcess(); + logger.info("Created process #" + processId); + } else if (action == 1) { + createBaseDirectory(processId.toString()); + filesAndDirectoriesIterator = filesAndDirectories.iterator(); + } else if (filesAndDirectoriesIterator.hasNext()) { + Path relativeItemToCopy = filesAndDirectoriesIterator.next(); + if (relativeItemToCopy.toString().equals(META_FILE_NAME)) { + if (filesAndDirectoriesIterator.hasNext()) { + relativeItemToCopy = filesAndDirectoriesIterator.next(); + } else { + Process process = processService.getById(processId); + copyAndAdjustMetsFile(process); + processService.save(process); + } + } else { + copyDirectoryOrFile(relativeItemToCopy); + } + } else { + Process process = processService.getById(processId); + copyAndAdjustMetsFile(process); + processService.save(process); + } + } + + /** + * Creates a process in the database. The process title must have already + * been generated during validation. + * + * @return database number of the created process + */ + private Integer createDatabaseProcess() throws ProcessGenerationException, DataException { + ProcessGenerator processGenerator = new ProcessGenerator(); + processGenerator.generateProcess(template.getId(), project.getId()); + Process process = processGenerator.getGeneratedProcess(); + process.setTitle(title); + process.setBaseType(baseType); + processService.save(process); + for (Task task : process.getTasks()) { + taskService.save(task); + } + return process.getId(); + } + + /** + * Transfers and adapts the {@code meta.xml}. The new process number is + * stored, the links are updated with the new IDs of the imported children, + * and media files are added. + * + * @param process + * the newly created process + */ + private void copyAndAdjustMetsFile(Process process) + throws IOException, InvalidImagesException, MediaNotFoundException, DAOException, DataException { + + Workpiece workpiece = metsService.loadWorkpiece(sourceDir.resolve(META_FILE_NAME).toUri()); + workpiece.setId(processId.toString()); + Map linkedChildren = searchLinkedProcesses(workpiece.getLogicalStructure()); + for (Entry linkedChild : linkedChildren.entrySet()) { + LinkedMetsResource link = linkedChild.getValue().getLink(); + link.setLoctype("Kitodo.Production"); + ImportingProcess importedChildProcess = importingProcesses.get(linkedChild.getKey()); + link.setUri(processService.getProcessURI(importedChildProcess.processId)); + addLinkInDatabase(process, importedChildProcess.processId); + } + fileService.searchForMedia(process, workpiece); + Path outputMetsFile = outputDir.resolve(META_FILE_NAME); + metsService.saveWorkpiece(workpiece, outputMetsFile.toUri()); + logger.info("Wrote METS file " + outputMetsFile); + } + + private void addLinkInDatabase(Process parent, Integer childProcessId) throws DAOException, DataException { + Process child = processService.getById(childProcessId); + child.setParent(parent); + parent.getChildren().add(child); + processService.save(child); + } + + /** + * Processes a faulty process. The output folder will be created in the + * error target folder. The fault information file is written in it. And + * then all the content is copied over. + * + * @param action + * action to take. The processing does not take place in one run, + * but the function is called by the task thread once per step. + * This is necessary to ensure stopability and the progress + * display in the task manager. + */ + private void executeActionForErroreousProcess(int action) throws IOException { + if (action == 0) { + createBaseDirectory(this.directoryName); + } else if (action == 1) { + String nameOfErrorFile = Helper.getTranslation("errors").concat(".txt"); + Path errorFile = outputDir.resolve(nameOfErrorFile); + if (errors.isEmpty()) { + String message = Helper.getTranslation("kitodoScript.importProcesses.brokenRelatedProcess", + String.join(", ", sharedErroneousProcesses)); + Files.writeString(errorFile, message, StandardCharsets.UTF_8); + } else { + Files.write(errorFile, errors, StandardCharsets.UTF_8); + } + logger.info("Wrote errors file " + errorFile); + } else { + copyDirectoryOrFile(filesAndDirectories.get(action - 2)); + } + } + + /** + * Determines the number of levels of children below this process. + * + * @return number of levels of children below + */ + int childDepth() { + OptionalInt childrenMaxDepth = children.parallelStream().mapToInt(ImportingProcess::childDepth).max(); + if (childrenMaxDepth.isEmpty()) { + return 0; + } else { + return childrenMaxDepth.getAsInt() + 1; + } + } + + // === UTILITIES === + + /** + * Creates the directory for the output. + * + * @param directoryName + * the name for the directory + */ + private void createBaseDirectory(String directoryName) throws IOException { + outputDir = copyToRoot.resolve(directoryName); + Files.createDirectories(outputDir); + logger.info("Created process directory " + outputDir); + } + + /** + * Copies a directory (without content) or a file from the source directory + * to the output directory. + * + * @param relativeItem + * the relative path, relative to the respective root + */ + private void copyDirectoryOrFile(Path relativeItem) throws IOException { + Path sourceItem = sourceDir.resolve(relativeItem); + Path destinationItem = outputDir.resolve(relativeItem); + if (Files.isDirectory(sourceItem)) { + Files.createDirectories(destinationItem); + logger.info("Created directory " + destinationItem); + } else { + Files.copy(sourceItem, destinationItem, StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.COPY_ATTRIBUTES, LinkOption.NOFOLLOW_LINKS); + logger.info("Copied " + sourceItem + " as " + destinationItem); + } + } + + void setProject(Project project) { + this.project = project; + } + + void setTemplate(Template template) { + this.template = template; + } + + @Override + public String toString() { + return "Importing process " + sourceDir; + } +} diff --git a/Kitodo/src/main/java/org/kitodo/production/services/command/KitodoScriptService.java b/Kitodo/src/main/java/org/kitodo/production/services/command/KitodoScriptService.java index ae2a73688f5..ac4abe05ba1 100644 --- a/Kitodo/src/main/java/org/kitodo/production/services/command/KitodoScriptService.java +++ b/Kitodo/src/main/java/org/kitodo/production/services/command/KitodoScriptService.java @@ -229,6 +229,18 @@ private boolean executeRemainingScript(List processes, String script) case "searchForMedia": searchForMedia(processes); break; + case "importProcesses": + String indir = parameters.get("indir"); + String project = parameters.get("project"); + String template = parameters.get("template"); + String errors = parameters.get("errors"); + try { + TaskManager.addTask(new ImportProcesses(indir, project, template, errors)); + Helper.setMessage("kitodoScript.importProcesses.executesInTaskManager"); + } catch (IllegalArgumentException e) { + Helper.setErrorMessage(e.getMessage()); + } + break; default: Helper.setErrorMessage("Unknown action", " - use: 'action:addRole, action:setTaskProperty, action:setStepStatus, " diff --git a/Kitodo/src/main/java/org/kitodo/production/services/validation/MetadataValidationService.java b/Kitodo/src/main/java/org/kitodo/production/services/validation/MetadataValidationService.java index e27a83060b4..a8014978b60 100644 --- a/Kitodo/src/main/java/org/kitodo/production/services/validation/MetadataValidationService.java +++ b/Kitodo/src/main/java/org/kitodo/production/services/validation/MetadataValidationService.java @@ -170,16 +170,36 @@ public ValidationResult validate(URI metsFileUri, URI rulesetFileUri) { * @param ruleset * Ruleset file * @return the validation result - * @throws DataException - * if an error occurs while reading from the search engine + * @throws DAOException + * if an error occurs while reading from the database */ public ValidationResult validate(Workpiece workpiece, RulesetManagementInterface ruleset) throws DAOException { + return validate(workpiece, ruleset, true); + } + + /** + * Validates a workpiece based on a rule set. + * + * @param workpiece + * METS file + * @param ruleset + * Ruleset file + * @param strict + * whether to validate document ID and presence of images + * @return the validation result + * @throws DAOException + * if an error occurs while reading from the database + */ + public ValidationResult validate(Workpiece workpiece, RulesetManagementInterface ruleset, boolean strict) + throws DAOException { Collection results = new ArrayList<>(); - results.add(checkTheIdentifier(workpiece)); + if (strict) { + results.add(checkTheIdentifier(workpiece)); + } results.add(metadataValidation.validate(workpiece, ruleset, getMetadataLanguage(), - getTranslations())); + getTranslations(), strict)); return merge(results); } @@ -189,8 +209,8 @@ public ValidationResult validate(Workpiece workpiece, RulesetManagementInterface * @param workpiece * METS file * @return the validation result - * @throws DataException - * if an error occurs while reading from the search engine + * @throws DAOException + * if an error occurs while reading from the database */ private ValidationResult checkTheIdentifier(Workpiece workpiece) throws DAOException { boolean error = false; diff --git a/Kitodo/src/main/resources/messages/errors_de.properties b/Kitodo/src/main/resources/messages/errors_de.properties index 359887cf370..7f798275246 100644 --- a/Kitodo/src/main/resources/messages/errors_de.properties +++ b/Kitodo/src/main/resources/messages/errors_de.properties @@ -10,6 +10,7 @@ # error=Fehler... +errors=Fehler # A errorAdding=Fehler beim Hinzuf\u00FCgen von ''{0}'' @@ -88,6 +89,21 @@ invalidIdentifierSame=Das Identifikationsmerkmal {0} enth\u00E4lt in {1} und {2} # J # K +kitodoScript.importProcesses.brokenRelatedProcess=Vorgang wurde wegen Fehlern in verkn\u00FCpften Vorg\u00E4ngen {0} nicht importiert. +kitodoScript.importProcesses.errors.cannotWrite=Fehlende Schreibberechtigung im Verzeichniss ''errors''! +kitodoScript.importProcesses.errors.isNoDirectory=''errors'' ist kein existierendes Verzeichnis! +kitodoScript.importProcesses.executesInTaskManager=Das KitodoSkript wird im Task Manager ausgef\u00FChrt. +kitodoScript.importProcesses.indir.isNull=Parameter ''indir'' fehlt! +kitodoScript.importProcesses.indir.isNoDirectory=''indir'' ist kein existierendes Verzeichnis! +kitodoScript.importProcesses.indir.cannotExecute=Der Inhalt des Importverzeichnisses kann nicht abgerufen werden (fehlende Berechtigung \u201Cexecute\u201D) +kitodoScript.importProcesses.missingChildren=Vorgang {0} enth\u004E Verweise auf fehlendene Kindvorg\u004Enge {1} +kitodoScript.importProcesses.noProcessTitleRule=Regelsatz gibt keine Bildungsvorschrift für den ''processTitle'' für Division {0} an! +kitodoScript.importProcesses.project.isNull=Parameter ''project'' fehlt! +kitodoScript.importProcesses.project.isNoProjectID=''project'' ist keine g\u00FCltige Projekt-ID +kitodoScript.importProcesses.project.noProjectWithID=Ein solches Projekt existiert nicht! +kitodoScript.importProcesses.template.isNull=Parameter ''template'' fehlt! +kitodoScript.importProcesses.template.isNoTemplateID=''template'' ist keine g\u00F6ltige Vorlagen-ID +kitodoScript.importProcesses.template.noTemplateWithID=Eine solche Produktionsvorlage existiert nicht! # L errorLoadingDocTypes=Regelsatz Konfigurationsfehler: Kein DocType ('division') gefunden diff --git a/Kitodo/src/main/resources/messages/errors_en.properties b/Kitodo/src/main/resources/messages/errors_en.properties index 9ed5f8fc905..2929d7d7366 100644 --- a/Kitodo/src/main/resources/messages/errors_en.properties +++ b/Kitodo/src/main/resources/messages/errors_en.properties @@ -10,6 +10,7 @@ # error=Error... +errors=Errors # A errorAdding=Error adding ''{0}'' @@ -88,6 +89,21 @@ invalidIdentifierSame=The identifier {0} has the same value in {1} and {2}. # J # K +kitodoScript.importProcesses.brokenRelatedProcess=Process was not imported due to errors in related process(es): {0} +kitodoScript.importProcesses.errors.cannotWrite=Cannot write in location ''errors''! +kitodoScript.importProcesses.errors.isNoDirectory=''errors'' is not a valid directory! +kitodoScript.importProcesses.executesInTaskManager=Executing Kitodo Script in the Task Manager. +kitodoScript.importProcesses.indir.isNull=Missing value for ''indir''! +kitodoScript.importProcesses.indir.isNoDirectory=''indir'' is not a valid directory! +kitodoScript.importProcesses.indir.cannotExecute=Cannot list ''indir'' directory (missing execute permission) +kitodoScript.importProcesses.missingChildren=Process {0} contains references to missing child processes {1} +kitodoScript.importProcesses.noProcessTitleRule=Ruleset does not specify a ''processTitle'' rule for division {0} +kitodoScript.importProcesses.project.isNull=Missing value for ''project''! +kitodoScript.importProcesses.project.isNoProjectID=''project'' is not int! +kitodoScript.importProcesses.project.noProjectWithID=There is no project with that ID +kitodoScript.importProcesses.template.isNull=Missing value for ''template''! +kitodoScript.importProcesses.template.isNoTemplateID=''template'' is not int! +kitodoScript.importProcesses.template.noTemplateWithID=There is no production template with that ID # L errorLoadingDocTypes=Ruleset configuration error: No DocType ('division') found diff --git a/Kitodo/src/main/resources/messages/errors_es.properties b/Kitodo/src/main/resources/messages/errors_es.properties index 605c55f6580..ad58615820e 100644 --- a/Kitodo/src/main/resources/messages/errors_es.properties +++ b/Kitodo/src/main/resources/messages/errors_es.properties @@ -10,6 +10,7 @@ # error=error... +errors=Errores # A errorAdding=Error al añadir ''{0}'' @@ -87,6 +88,21 @@ invalidIdentifierSame=La característica de identificación {0} contiene el mism # J # K +kitodoScript.importProcesses.brokenRelatedProcess=Proceso no se agrega debido a errores en los procesos relaciónados {0}. +kitodoScript.importProcesses.errors.isNoDirectory=¡''errors'' no es una lista de archivos! +kitodoScript.importProcesses.errors.cannotWrite=¡No puedo describir el lugar ''errors''! +kitodoScript.importProcesses.executesInTaskManager=Voy a llevar al guión de Kitodo al administrador de tareas pendientes. +kitodoScript.importProcesses.indir.isNull=¡Falta la variable ''indir''! +kitodoScript.importProcesses.indir.isNoDirectory=¡''indir'' no es una lista de archivos! +kitodoScript.importProcesses.indir.cannotExecute=No se puede obtener la lista del directorio ''indir'' (falta de permiso para ejecutar) +kitodoScript.importProcesses.missingChildren=El proceso {0} contiene referencias al proceso {1} del niño perdido. +kitodoScript.importProcesses.noProcessTitleRule=Declaración de reglas no especifica una regla de de educación ''processTitle'' para la division {0} +kitodoScript.importProcesses.project.isNull=¡Falta la variable ''project''! +kitodoScript.importProcesses.project.isNoProjectID=''project'' no es un ID válido de proyecto (debe ser entero) +kitodoScript.importProcesses.project.noProjectWithID=¡Tal proyecto no existe! +kitodoScript.importProcesses.template.isNull=¡Falta la variable ''template''! +kitodoScript.importProcesses.template.isNoTemplateID=¡''template'' no es un ID válido en la plantilla! +kitodoScript.importProcesses.template.noTemplateWithID=No existe tal plantilla con estos dígitos de identificación ''template'' # L errorLoadingDocTypes=Error de configuración del conjunto de reglas: No se ha encontrado ningún DocType ('division') diff --git a/Kitodo/src/main/resources/messages/messages_de.properties b/Kitodo/src/main/resources/messages/messages_de.properties index 4e8d28a0369..977d2061390 100644 --- a/Kitodo/src/main/resources/messages/messages_de.properties +++ b/Kitodo/src/main/resources/messages/messages_de.properties @@ -645,6 +645,7 @@ importingData=Daten werden importiert... importChildren=Kindvorg\u00E4nge importieren importDepth=Importtiefe importDms=Export in das DMS +importProcesses=Vorg\u00E4nge importieren imprint=Impressum # used in "LegalTexts.java" imprintDefaultText=Das Impressum f\u00FCr dieses System wurde noch nicht hinterlegt. diff --git a/Kitodo/src/main/resources/messages/messages_en.properties b/Kitodo/src/main/resources/messages/messages_en.properties index c38c291cedc..d95a8ed8ffd 100644 --- a/Kitodo/src/main/resources/messages/messages_en.properties +++ b/Kitodo/src/main/resources/messages/messages_en.properties @@ -646,6 +646,7 @@ importingData=Data are being imported importChildren=Import child processes importDepth=Import depth importDms=Import into DMS +importProcesses=Import processes imprint=Imprint # used in "LegalTexts.java" imprintDefaultText=Imprint information have not been configured for this system. diff --git a/Kitodo/src/main/resources/messages/messages_es.properties b/Kitodo/src/main/resources/messages/messages_es.properties index 6d8b2c01fcd..0effd29d4bc 100644 --- a/Kitodo/src/main/resources/messages/messages_es.properties +++ b/Kitodo/src/main/resources/messages/messages_es.properties @@ -639,6 +639,7 @@ importingData=Los datos se importan... importChildren=Importar procesos hijos importDepth=Profundidad de importación importDms=Exportación a DMS +importProcesses=Importacione de procesos imprint=Pie de imprenta # used in "LegalTexts.java" imprintDefaultText=La huella de este sistema aún no se ha almacenado. diff --git a/Kitodo/src/main/webapp/WEB-INF/templates/includes/processes/executeScriptSelectedPopup.xhtml b/Kitodo/src/main/webapp/WEB-INF/templates/includes/processes/executeScriptSelectedPopup.xhtml index 1207f4ad990..9f82edc96a0 100644 --- a/Kitodo/src/main/webapp/WEB-INF/templates/includes/processes/executeScriptSelectedPopup.xhtml +++ b/Kitodo/src/main/webapp/WEB-INF/templates/includes/processes/executeScriptSelectedPopup.xhtml @@ -89,6 +89,9 @@ +

diff --git a/Kitodo/src/test/java/org/kitodo/production/services/command/ImportProcessesIT.java b/Kitodo/src/test/java/org/kitodo/production/services/command/ImportProcessesIT.java new file mode 100644 index 00000000000..30630c31157 --- /dev/null +++ b/Kitodo/src/test/java/org/kitodo/production/services/command/ImportProcessesIT.java @@ -0,0 +1,332 @@ +/* + * (c) Kitodo. Key to digital objects e. V. + * + * This file is part of the Kitodo project. + * + * It is licensed under GNU General Public License version 3 or later. + * + * For the full copyright and license information, please read the + * GPL3-License.txt file that was distributed with this source code. + */ + +package org.kitodo.production.services.command; + +// abbreviations +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +// base Java +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Date; +import java.time.LocalDate; +import java.time.ZoneId; + +// open source code +import org.apache.commons.lang3.SystemUtils; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.kitodo.ExecutionPermission; +import org.kitodo.MockDatabase; +import org.kitodo.SecurityTestUtils; +import org.kitodo.TreeDeleter; +import org.kitodo.config.ConfigCore; +import org.kitodo.config.enums.ParameterCore; +import org.kitodo.data.database.beans.Client; +import org.kitodo.data.database.beans.Folder; +import org.kitodo.data.database.beans.Process; +import org.kitodo.data.database.beans.Ruleset; +import org.kitodo.data.database.beans.Task; +import org.kitodo.data.database.beans.Template; +import org.kitodo.data.database.beans.User; +import org.kitodo.production.services.ServiceManager; +import org.kitodo.test.utils.ProcessTestUtils; + +public class ImportProcessesIT { + private static final Path ERRORS_DIR_PATH = Paths.get("src/test/resources/errors"); + + private int firstProcessId, secondProcessId, thirdProcessId; + private static Template template; + + @BeforeClass + public static void prepareDatabase() throws Exception { + MockDatabase.startNode(); + MockDatabase.insertProcessesFull(); + + // add ruleset + Ruleset ruleset = new Ruleset(); + ruleset.setTitle("ImportProcessesIT"); + ruleset.setFile("ImportProcessesIT.xml"); + ruleset.setOrderMetadataByRuleset(true); + Client clientOne = ServiceManager.getClientService().getById(1); + ruleset.setClient(clientOne); + ServiceManager.getRulesetService().save(ruleset); + + Task task = new Task(); + task.getRoles().add(ServiceManager.getRoleService().getById(1)); + ServiceManager.getTaskService().save(task); + + template = new Template(); + template.setTitle("Import processes template"); + LocalDate localDate = LocalDate.of(2023, 8, 9); + template.setCreationDate(Date.from(localDate.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant())); + template.setClient(clientOne); + template.setDocket(ServiceManager.getDocketService().getById(1)); + template.getProjects().add(ServiceManager.getProjectService().getById(1)); + template.setRuleset(ruleset); + ServiceManager.getTemplateService().save(template, true); + + task.setTemplate(template); + template.getTasks().add(task); + ServiceManager.getTemplateService().save(template, true); + ServiceManager.getTaskService().save(task); + + Folder local = ServiceManager.getFolderService().getById(6); + local.setMimeType("image/jpeg"); + ServiceManager.getFolderService().saveToDatabase(local); + + User userOne = ServiceManager.getUserService().getById(1); + SecurityTestUtils.addUserDataToSecurityContext(userOne, 1); + } + + @BeforeClass + public static void setScriptPermission() throws Exception { + if (!SystemUtils.IS_OS_WINDOWS) { + ExecutionPermission + .setExecutePermission(new File(ConfigCore.getParameter(ParameterCore.SCRIPT_CREATE_DIR_META))); + } + } + + @Before + public void createOutputDirectories() throws Exception { + Files.createDirectories(ERRORS_DIR_PATH); + } + + /** + * Tests the target behavior specified. + */ + @Test + public void shouldImport() throws Exception { + // create test object + String indir = "src/test/resources/ImportProcessesIT"; + String projectId = "1"; + String templateId = template.getId().toString(); + String errors = "src/test/resources/errors"; + ImportProcesses underTest = new ImportProcesses(indir, projectId, templateId, errors); + + int processesBefore = ServiceManager.getProcessService().countDatabaseRows().intValue(); + + // initialize + underTest.run(0); + assertEquals("should require 43 actions", 43, underTest.totalActions); + + // validate: correct processes + underTest.run(1); + assertEquals("should have validated p1_valid", "p1_valid", underTest.validatingImportingProcess.directoryName); + ImportingProcess p1_valid = underTest.validatingImportingProcess; + assertTrue("should have validated without errors", p1_valid.errors.isEmpty()); + + underTest.run(2); + assertEquals("should have validated p1c1_valid", "p1c1_valid", + underTest.validatingImportingProcess.directoryName); + ImportingProcess p1c1_valid = underTest.validatingImportingProcess; + assertTrue("should have validated without errors", p1c1_valid.errors.isEmpty()); + + underTest.run(3); + assertEquals("should have validated p1c2_valid", "p1c2_valid", + underTest.validatingImportingProcess.directoryName); + ImportingProcess p1c2_valid = underTest.validatingImportingProcess; + assertTrue("should have validated without errors", p1c2_valid.errors.isEmpty()); + + assertTrue("p1_valid should be correct", p1_valid.isCorrect()); + assertTrue("p1c1_valid should be correct", p1c1_valid.isCorrect()); + assertTrue("p1c2_valid should be correct", p1c2_valid.isCorrect()); + + // validate: error case 'parent is missing a child' + underTest.run(4); + assertEquals("should have validated p2_parentMissingAChild", "p2_parentMissingAChild", + underTest.validatingImportingProcess.directoryName); + ImportingProcess p2_parentMissingAChild = underTest.validatingImportingProcess; + assertFalse("should have validated with error", p2_parentMissingAChild.errors.isEmpty()); + + underTest.run(5); + assertEquals("should have validated p2c1_valid", "p2c1_valid", + underTest.validatingImportingProcess.directoryName); + ImportingProcess p2c1_valid = underTest.validatingImportingProcess; + assertTrue("should have validated without errors", p2c1_valid.errors.isEmpty()); + + assertFalse("p1_valid should not be correct", p2_parentMissingAChild.isCorrect()); + assertFalse("p1c1_valid should not be correct, due to problem in parent case", p2c1_valid.isCorrect()); + + // validate: error case 'parent not valid' + underTest.run(6); + assertEquals("should have validated p3_not-valid", "p3_not-valid", + underTest.validatingImportingProcess.directoryName); + ImportingProcess p3_notValid = underTest.validatingImportingProcess; + assertFalse("should have validated with error", p3_notValid.errors.isEmpty()); + + underTest.run(7); + assertEquals("should have validated p3c1_valid", "p3c1_valid", + underTest.validatingImportingProcess.directoryName); + ImportingProcess p3c1_valid = underTest.validatingImportingProcess; + assertTrue("should have validated without errors", p3c1_valid.errors.isEmpty()); + + assertFalse("p1_valid should not be correct", p3_notValid.isCorrect()); + assertFalse("p1c1_valid should not be correct, due to problem in parent case", p3c1_valid.isCorrect()); + + // validate: error case 'child not valid' + underTest.run(8); + assertEquals("should have validated p4_valid_butChildIsNot", "p4_valid_butChildIsNot", + underTest.validatingImportingProcess.directoryName); + ImportingProcess p4_valid_butChildIsNot = underTest.validatingImportingProcess; + assertTrue("should have validated without errors", p4_valid_butChildIsNot.errors.isEmpty()); + + underTest.run(9); + assertEquals("should have validated p4c1_not-valid", "p4c1_not-valid", + underTest.validatingImportingProcess.directoryName); + ImportingProcess p4c1_notValid = underTest.validatingImportingProcess; + assertFalse("should have validated with error", p4c1_notValid.errors.isEmpty()); + + assertFalse("p1_valid should not be correct, due to problem in child case", p4_valid_butChildIsNot.isCorrect()); + assertFalse("p1c1_valid should not be correct", p4c1_notValid.isCorrect()); + + // copy files and create database entry + + // p1c1_valid (OK) + firstProcessId = processesBefore + 1; + Path processPath = Paths.get("src/test/resources/metadata", Integer.toString(firstProcessId)); + Path imagesPath = processPath.resolve("images"); + Path mediaPath = imagesPath.resolve("17_123_0001_media"); + Path imageOne = mediaPath.resolve("00000001.jpg"); + Path metaXml = processPath.resolve("meta.xml"); + + underTest.run(10); + assertEquals("should have created 1st process,", Long.valueOf(firstProcessId), + ServiceManager.getProcessService().countDatabaseRows()); + underTest.run(11); + assertTrue("should have created process directory", Files.isDirectory(processPath)); + underTest.run(12); + underTest.run(13); + underTest.run(14); + underTest.run(15); + + assertTrue("should have created images directory", Files.isDirectory(imagesPath)); + assertTrue("should have created media directory", Files.isDirectory(mediaPath)); + assertTrue("should have copied media file", Files.exists(imageOne)); + assertTrue("should have written meta.xml file", Files.exists(metaXml)); + assertTrue("should have added image to meta.xml file", + Files.readString(metaXml, UTF_8).contains("xlink:href=\"images/17_123_0001_media/00000001.jpg\"")); + + // p1c2_valid (OK) + secondProcessId = processesBefore + 2; + processPath = Paths.get("src/test/resources/metadata", Integer.toString(secondProcessId)); + metaXml = processPath.resolve("meta.xml"); + + underTest.run(16); + underTest.run(17); + underTest.run(18); + underTest.run(19); + underTest.run(20); + underTest.run(21); + assertTrue("should have added image to meta.xml file", + Files.readString(metaXml, UTF_8).contains("xlink:href=\"images/17_123_0002_media/00000001.jpg\"")); + + // p2c1_valid (error, broken parent) + underTest.run(22); + assertTrue("should have error directory", Files.isDirectory(Paths.get("src/test/resources/errors/p2c1_valid"))); + underTest.run(23); + assertTrue("should have written Errors.txt file", + Files.exists(Paths.get("src/test/resources/errors/p2c1_valid/Errors.txt"))); + assertTrue("should have written error message", + Files.readString(Paths.get("src/test/resources/errors/p2c1_valid/Errors.txt"), UTF_8) + .contains("errors in related process(es): p2_parentMissingAChild")); + underTest.run(24); + assertTrue("should have copied meta.xml file", + Files.exists(Paths.get("src/test/resources/errors/p2c1_valid/meta.xml"))); + + // p3c1_valid (error, broken parent) + underTest.run(25); + underTest.run(26); + assertTrue("should have written error message", + Files.readString(Paths.get("src/test/resources/errors/p3c1_valid/Errors.txt"), UTF_8) + .contains("errors in related process(es): p3_not-valid")); + underTest.run(27); + assertTrue("should have copied meta.xml file", + Files.exists(Paths.get("src/test/resources/errors/p3c1_valid/meta.xml"))); + + // p4c1_not-valid (error, not valid) + underTest.run(28); + underTest.run(29); + assertTrue("should have written error message", + Files.readString(Paths.get("src/test/resources/errors/p4c1_not-valid/Errors.txt"), UTF_8) + .contains("Validation error")); + underTest.run(30); + + // p1_valid (OK) + thirdProcessId = processesBefore + 3; + processPath = Paths.get("src/test/resources/metadata", Integer.toString(thirdProcessId)); + metaXml = processPath.resolve("meta.xml"); + + underTest.run(31); + assertEquals("should have created 3rd process,", processesBefore + 3, + (long) ServiceManager.getProcessService().countDatabaseRows()); + underTest.run(32); + assertTrue("should have created process directory", Files.isDirectory(processPath)); + underTest.run(33); + String thirdMetaXml = Files.readString(metaXml, UTF_8); + assertThat("should have added correct child links to meta.xml file", thirdMetaXml, + containsString("xlink:href=\"database://?process.id=" + firstProcessId + "\"")); + assertThat("should have added correct child links to meta.xml file", thirdMetaXml, + containsString("xlink:href=\"database://?process.id=" + secondProcessId + "\"")); + + Process parent = ServiceManager.getProcessService().getById(6); + assertEquals("parent should have 2 children", 2, parent.getChildren().size()); + assertEquals("child (ID " + firstProcessId + ") should have the correct parent", parent, + ServiceManager.getProcessService().getById(firstProcessId).getParent()); + assertEquals("child (ID " + secondProcessId + ") should have the correct parent", parent, + ServiceManager.getProcessService().getById(secondProcessId).getParent()); + + // p2_parentMissingAChild (error) + underTest.run(34); + underTest.run(35); + underTest.run(36); + + // p3_not-valid (error) + underTest.run(37); + underTest.run(38); + underTest.run(39); + + // p4_valid_butChildIsNot (error) + underTest.run(40); + underTest.run(41); + underTest.run(42); + + // import results + assertEquals("Should import 3 processes,", processesBefore + 3, + (long) ServiceManager.getProcessService().countDatabaseRows()); + assertEquals("Should not import 6 processes,", 6, ERRORS_DIR_PATH.toFile().list().length); + } + + @After + public void deleteCreatedFiles() throws Exception { + ProcessTestUtils.removeTestProcess(firstProcessId); + ProcessTestUtils.removeTestProcess(secondProcessId); + ProcessTestUtils.removeTestProcess(thirdProcessId); + TreeDeleter.deltree(ERRORS_DIR_PATH); + } + + @AfterClass + public static void cleanDatabase() throws Exception { + MockDatabase.stopNode(); + MockDatabase.cleanDatabase(); + } +} diff --git a/Kitodo/src/test/resources/ImportProcessesIT/p1_valid/meta.xml b/Kitodo/src/test/resources/ImportProcessesIT/p1_valid/meta.xml new file mode 100644 index 00000000000..a597246e01d --- /dev/null +++ b/Kitodo/src/test/resources/ImportProcessesIT/p1_valid/meta.xml @@ -0,0 +1,20 @@ + + + + +17 +123 +Lorem ipsum dolor sit amet + + Molenstraat + 21 + B + + + +
+ +
+
+ +
 diff --git a/Kitodo/src/test/resources/ImportProcessesIT/p1c1_valid/images/17_123_0001_media/00000001.jpg b/Kitodo/src/test/resources/ImportProcessesIT/p1c1_valid/images/17_123_0001_media/00000001.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6becb4c508faf6b85146b62a7811ecdf617e2482 GIT binary patch literal 1155 zcmex= zRY=j$kxe)-kzJ`!#HexNLJno8jR!@8E`CrkPAY2R|V^&07y2J$~}^+4C1KUw!=a z`ODXD-+%o41@ado12e>1aG#<1OAzQUCSV+}u!H=?$W#u*%z`YeiiT`Lj)Clng~Cck zjT|CQ6Blkg$f;}`^g%SK=pvVxipfLOk07sseMX$en#l4Q++zrT-D2QjW&}navmk># z!{0xPx-=N5h1c`^-LXI4p7-AoJE6<7KD&Oj@2yFHR9G-i@JsnTlqJmx>cQIEiV(iLq^tDosjEm?Kr3A5k{_WR7AM9=oF)GARA zvK6i``nSv8c`+x)xjl)xv%TNjT>i&-@poeV;ktiE?HGO>{?EYi{*O}qhdDPrEB{XX z_Mc(<-|m!8hqs^m^yvP4ealbR-zUqY?OnG|<3B@lN&Qo<`3&`MI1B$XEd6f!ZEsb@ ztHR}-EB&N3P4@YGHV{<}=Ji|hHBWPYSW}nAy5Kb$i@F%lvG%e53_1Qcnd)zSKRhdK zru@O+TA~FKf1p+PVd9qzFD`@=0036`s$l`RoIlZ22)P56ht04 pt@ie?YR8fO + + + +0001 +Lorem ipsum dolor sit amet +07.08.2023 13:31 + + + +
+ diff --git a/Kitodo/src/test/resources/ImportProcessesIT/p1c2_valid/images/17_123_0002_media/00000001.jpg b/Kitodo/src/test/resources/ImportProcessesIT/p1c2_valid/images/17_123_0002_media/00000001.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6becb4c508faf6b85146b62a7811ecdf617e2482 GIT binary patch literal 1155 zcmex= zRY=j$kxe)-kzJ`!#HexNLJno8jR!@8E`CrkPAY2R|V^&07y2J$~}^+4C1KUw!=a z`ODXD-+%o41@ado12e>1aG#<1OAzQUCSV+}u!H=?$W#u*%z`YeiiT`Lj)Clng~Cck zjT|CQ6Blkg$f;}`^g%SK=pvVxipfLOk07sseMX$en#l4Q++zrT-D2QjW&}navmk># z!{0xPx-=N5h1c`^-LXI4p7-AoJE6<7KD&Oj@2yFHR9G-i@JsnTlqJmx>cQIEiV(iLq^tDosjEm?Kr3A5k{_WR7AM9=oF)GARA zvK6i``nSv8c`+x)xjl)xv%TNjT>i&-@poeV;ktiE?HGO>{?EYi{*O}qhdDPrEB{XX z_Mc(<-|m!8hqs^m^yvP4ealbR-zUqY?OnG|<3B@lN&Qo<`3&`MI1B$XEd6f!ZEsb@ ztHR}-EB&N3P4@YGHV{<}=Ji|hHBWPYSW}nAy5Kb$i@F%lvG%e53_1Qcnd)zSKRhdK zru@O+TA~FKf1p+PVd9qzFD`@=0036`s$l`RoIlZ22)P56ht04 pt@ie?YR8fO + + + +0002 +Lorem ipsum dolor sit amet +07.08.2023 13:35 + + + +
+ diff --git a/Kitodo/src/test/resources/ImportProcessesIT/p2_parentMissingAChild/meta.xml b/Kitodo/src/test/resources/ImportProcessesIT/p2_parentMissingAChild/meta.xml new file mode 100644 index 00000000000..c803d920917 --- /dev/null +++ b/Kitodo/src/test/resources/ImportProcessesIT/p2_parentMissingAChild/meta.xml @@ -0,0 +1,20 @@ + + + + +17 +123 +Lorem ipsum dolor sit amet + + Molenstraat + 21 + B + + + +
+ +
+
+ +
 diff --git a/Kitodo/src/test/resources/ImportProcessesIT/p2c1_valid/meta.xml b/Kitodo/src/test/resources/ImportProcessesIT/p2c1_valid/meta.xml new file mode 100644 index 00000000000..4cc7eaf827f --- /dev/null +++ b/Kitodo/src/test/resources/ImportProcessesIT/p2c1_valid/meta.xml @@ -0,0 +1,12 @@ + + + + +0002 +Lorem ipsum dolor sit amet +07.08.2023 13:35 + + + +
+ diff --git a/Kitodo/src/test/resources/ImportProcessesIT/p3_not-valid/meta.xml b/Kitodo/src/test/resources/ImportProcessesIT/p3_not-valid/meta.xml new file mode 100644 index 00000000000..b5317e24b8d --- /dev/null +++ b/Kitodo/src/test/resources/ImportProcessesIT/p3_not-valid/meta.xml @@ -0,0 +1,14 @@ + + + + +47 +11 +Bar + + +
+ +
+ +
 diff --git a/Kitodo/src/test/resources/ImportProcessesIT/p3c1_valid/meta.xml b/Kitodo/src/test/resources/ImportProcessesIT/p3c1_valid/meta.xml new file mode 100644 index 00000000000..4d1b57e258f --- /dev/null +++ b/Kitodo/src/test/resources/ImportProcessesIT/p3c1_valid/meta.xml @@ -0,0 +1,12 @@ + + + + +0001 +Lorem ipsum dolor sit amet +07.08.2023 13:31 + + + +
+ diff --git a/Kitodo/src/test/resources/ImportProcessesIT/p4_valid_butChildIsNot/meta.xml b/Kitodo/src/test/resources/ImportProcessesIT/p4_valid_butChildIsNot/meta.xml new file mode 100644 index 00000000000..3104b0433ae --- /dev/null +++ b/Kitodo/src/test/resources/ImportProcessesIT/p4_valid_butChildIsNot/meta.xml @@ -0,0 +1,14 @@ + + + + +1234 +567 +Lorem ipsum dolor sit amet + + +
+ +
+ +
 diff --git a/Kitodo/src/test/resources/ImportProcessesIT/p4c1_not-valid/meta.xml b/Kitodo/src/test/resources/ImportProcessesIT/p4c1_not-valid/meta.xml new file mode 100644 index 00000000000..ae6653bb878 --- /dev/null +++ b/Kitodo/src/test/resources/ImportProcessesIT/p4c1_not-valid/meta.xml @@ -0,0 +1,11 @@ + + + + +0003 +li cloblwl Shbal + + + +
+ diff --git a/Kitodo/src/test/resources/rulesets/ImportProcessesIT.xml b/Kitodo/src/test/resources/rulesets/ImportProcessesIT.xml new file mode 100644 index 00000000000..cbd10098727 --- /dev/null +++ b/Kitodo/src/test/resources/rulesets/ImportProcessesIT.xml @@ -0,0 +1,100 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + [1-9]\d* + + + + + [1-9]\d{0,4} + + + + \d{4} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +