diff --git a/Kitodo-API/src/main/java/org/kitodo/api/validation/metadata/MetadataValidationInterface.java b/Kitodo-API/src/main/java/org/kitodo/api/validation/metadata/MetadataValidationInterface.java index 375fffb7e38..10b927bfcf1 100644 --- a/Kitodo-API/src/main/java/org/kitodo/api/validation/metadata/MetadataValidationInterface.java +++ b/Kitodo-API/src/main/java/org/kitodo/api/validation/metadata/MetadataValidationInterface.java @@ -67,8 +67,10 @@ ValidationResult validate(URI metsFileUri, URI rulesetFileUri, List metadataLanguage, Map translations); + List metadataLanguage, Map translations, boolean checkMedia); } diff --git a/Kitodo-Validation/src/main/java/org/kitodo/validation/metadata/MetadataValidation.java b/Kitodo-Validation/src/main/java/org/kitodo/validation/metadata/MetadataValidation.java index 5001e8f233c..179dd4edb6d 100644 --- a/Kitodo-Validation/src/main/java/org/kitodo/validation/metadata/MetadataValidation.java +++ b/Kitodo-Validation/src/main/java/org/kitodo/validation/metadata/MetadataValidation.java @@ -106,7 +106,7 @@ public ValidationResult validate(URI metsFileUri, URI rulesetFileUri, List metadataLanguage, Map translations) { + List metadataLanguage, Map translations, boolean checkMedia) { Collection results = new ArrayList<>(); - results.add(checkForStructuresWithoutMedia(workpiece, translations)); - results.add(checkForUnlinkedMedia(workpiece, translations)); + if (checkMedia) { + results.add(checkForStructuresWithoutMedia(workpiece, translations)); + results.add(checkForUnlinkedMedia(workpiece, translations)); + } for (LogicalDivision logicalDivision : workpiece.getAllLogicalDivisions()) { results.addAll(checkMetadataRules(logicalDivision.toString(), logicalDivision.getType(), diff --git a/Kitodo/src/main/java/org/kitodo/production/helper/Helper.java b/Kitodo/src/main/java/org/kitodo/production/helper/Helper.java index 955cd7c66f3..92e90b6eb14 100644 --- a/Kitodo/src/main/java/org/kitodo/production/helper/Helper.java +++ b/Kitodo/src/main/java/org/kitodo/production/helper/Helper.java @@ -489,6 +489,9 @@ public static String getTranslation(String title, String... insertions) { private static String appendUnusedInsertions(String message, String... insertions) { StringBuilder messageBuilder = new StringBuilder(message); for (String insertion : insertions) { + if (Objects.isNull(insertion)) { + continue; + } String separator = ": "; insertion = Objects.toString(insertion); if (!messageBuilder.toString().contains(insertion)) { diff --git a/Kitodo/src/main/java/org/kitodo/production/services/command/ImportProcesses.java b/Kitodo/src/main/java/org/kitodo/production/services/command/ImportProcesses.java new file mode 100644 index 00000000000..ddd4dd19605 --- /dev/null +++ b/Kitodo/src/main/java/org/kitodo/production/services/command/ImportProcesses.java @@ -0,0 +1,347 @@ +/* + * (c) Kitodo. Key to digital objects e. V. + * + * This file is part of the Kitodo project. + * + * It is licensed under GNU General Public License version 3 or later. + * + * For the full copyright and license information, please read the + * GPL3-License.txt file that was distributed with this source code. + */ + +package org.kitodo.production.services.command; + +// base Java +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Comparator; +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +// open source code +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.kitodo.api.dataeditor.rulesetmanagement.RulesetManagementInterface; +import org.kitodo.config.ConfigCore; +import org.kitodo.config.KitodoConfig; +import org.kitodo.config.enums.ParameterCore; +import org.kitodo.data.database.beans.Project; +import org.kitodo.data.database.beans.Template; +import org.kitodo.data.database.exceptions.DAOException; +import org.kitodo.data.exceptions.DataException; +import org.kitodo.exceptions.InvalidImagesException; +import org.kitodo.exceptions.MediaNotFoundException; +import org.kitodo.exceptions.ProcessGenerationException; +import org.kitodo.production.helper.Helper; +import org.kitodo.production.helper.tasks.EmptyTask; +import org.kitodo.production.services.ServiceManager; +import org.kitodo.production.services.data.ProjectService; +import org.kitodo.production.services.data.TemplateService; + + +/** + * Long-running task importing processes into Production. + */ +public final class ImportProcesses extends EmptyTask { + private static final Logger logger = LogManager.getLogger(ImportProcesses.class); + + // number of actions required for initialization + private static final int INIT_ACTIONS_COUNT = 1; + // number of actions required for validation + private static final int VALIDATION_ACTIONS_COUNT = 1; + + // services required + private final ProjectService projectService = ServiceManager.getProjectService(); + private final TemplateService templateService = ServiceManager.getTemplateService(); + + // global system configuration + private final boolean strictValidation = ConfigCore.getBooleanParameter(ParameterCore.VALIDATION_FAIL_ON_WARNING); + + // data + private final Path importRootPath; + private final Project project; + private final Template templateForProcesses; + private RulesetManagementInterface ruleset; + private final Path errorPath; + private final TreeMap importingProcesses; + private final int numberOfImportingProcesses; + + // thread execution + private int step = 0; + int totalActions = 1; + private Iterator importingProcessesIterator; + ImportingProcess validatingImportingProcess; + private ImportingProcess currentlyImporting; + private int nextAction = 0; + private int numberOfRemainingActions = 0; + + + /** + * Constructor. Creates a {@code ProcessesImport} + * long-running task. + * + * @param indir + * input by the user for the source root directory. Can + * [invalidly] be {@code null} if the user has not specified it. + * @param project + * input by the user for the project ID. Can [invalidly] be + * {@code null} if the user has not specified it. + * @param template + * input by the for the process template ID. Can [invalidly] be + * {@code null} if the user has not specified it. + * @param errors + * input by the user for the source root directory. Can [validly] + * be {@code null} if the user has not specified it. + * @throws IllegalArgumentException + * if the user hasn't specified all the necessary parameters, or + * if the parameters have unusable values. Exception message is + * a message key that must be resolved via message properties. + */ + public ImportProcesses(String indir, String project, String template, String errors) throws IOException { + super(indir); + this.importRootPath = checkIndir(indir); + this.project = checkProject(project); + this.templateForProcesses = checkTemplate(template); + this.errorPath = checkErrors(errors); + try (Stream pathStream = Files.walk(this.importRootPath, 1)) { + this.importingProcesses = pathStream.filter(Files::isDirectory) + .filter(Predicate.not(this.importRootPath::equals)) + .collect(Collectors.toMap(path -> path.getFileName().toString(), ImportingProcess::new, + (existing, replacing) -> replacing, TreeMap::new)); + } + this.numberOfImportingProcesses = importingProcesses.size(); + } + + /** + * Clone constructor. Creates a copy of the object. But + * because the object is a terminated thread, it makes a new thread that can + * be started again. + */ + private ImportProcesses(ImportProcesses source) { + super(source); + this.importRootPath = source.importRootPath; + this.project = source.project; + this.templateForProcesses = source.templateForProcesses; + this.ruleset = source.ruleset; + this.errorPath = source.errorPath; + this.importingProcesses = source.importingProcesses; + this.numberOfImportingProcesses = source.numberOfImportingProcesses; + + this.step = source.step; + this.totalActions = source.totalActions; + this.importingProcessesIterator = source.importingProcessesIterator; + this.validatingImportingProcess = source.validatingImportingProcess; + this.currentlyImporting = source.currentlyImporting; + this.nextAction = source.nextAction; + this.numberOfRemainingActions = source.numberOfRemainingActions; + } + + /** + * Checks whether the {@code indir} parameter is specified and valid. If + * not, a corresponding error message is thrown as an exception. (This is + * picked up above and will be translated and displayed to the user.) The + * {@code indir} directory must be specified, it must exist, and the Tomcat + * must have permission to get the directory listing (execution permission). + * + * @param indir + * directory entered by the user. Can be {@code null} if the + * parameter is not specified + * @return a Path object to the root directory for bulk import + */ + private Path checkIndir(String indir) { + if (Objects.isNull(indir)) { + throw new IllegalArgumentException("kitodoScript.importProcesses.indir.isNull"); + } + Path importRoot = Paths.get(indir); + if (!Files.isDirectory(importRoot)) { + throw new IllegalArgumentException("kitodoScript.importProcesses.indir.isNoDirectory"); + } + if (!Files.isExecutable(importRoot)) { + throw new IllegalArgumentException("kitodoScript.importProcesses.indir.cannotExecute"); + } + return importRoot; + } + + /** + * Checks whether the parameter {@code project} is specified and valid. If + * not, a corresponding error message is thrown as an exception. (This is + * picked up above and will be translated and displayed to the user.) The + * project must be specified, it must be syntactically valid (a positive + * integer), and a project with that ID must exist. + * + * @param project + * user-entered project number. Can be {@code null} if the + * parameter is not specified + * @return the project object from the database + */ + private Project checkProject(String project) { + if (Objects.isNull(project)) { + throw new IllegalArgumentException("kitodoScript.importProcesses.project.isNull"); + } + if (!project.matches("[\\d]+")) { + throw new IllegalArgumentException("kitodoScript.importProcesses.project.isNoProjectID"); + } + Integer projectInteger = Integer.valueOf(project); + try { + return projectService.getById(projectInteger); + } catch (DAOException e) { + logger.catching(e); + throw new IllegalArgumentException("kitodoScript.importProcesses.project.noProjectWithID"); + } + } + + /** + * Checks whether the parameter {@code template} is specified and valid. If + * not, a corresponding error message is thrown as an exception. (This is + * picked up above and will be translated and displayed to the user.) The + * production template must be specified, its ID must be syntactically valid + * (a positive integer), and a production template with that ID must exist. + * + * @param template + * user-entered production template number. Can be {@code null} + * if the parameter is not specified + * @return the production template object from the database + */ + private Template checkTemplate(String template) { + if (Objects.isNull(template)) { + throw new IllegalArgumentException("kitodoScript.importProcesses.template.isNull"); + } + if (!template.matches("[\\d]+")) { + throw new IllegalArgumentException("kitodoScript.importProcesses.template.isNoTemplateID"); + } + Integer templateInteger = Integer.valueOf(template); + try { + return templateService.getById(templateInteger); + } catch (DAOException e) { + logger.catching(e); + throw new IllegalArgumentException("kitodoScript.importProcesses.template.noTemplateWithID"); + } + } + + /** + * Checks whether the parameter {@code errors} is specified and—if so—is + * valid. If not, a corresponding error message is thrown as an exception. + * (This is picked up above and will be translated and displayed to the + * user.) The errors directory is optional, but if provided, it must exist + * and be writable by Tomcat. + * + * @param errors + * Path to directory for errors. May be {@code null} if not + * specified + * @return a Path object to the errors directory, or {@code null} if not + * specified + */ + private Path checkErrors(String errors) { + if (Objects.isNull(errors)) { + return null; + } + Path errorDir = Paths.get(errors); + if (!Files.isDirectory(errorDir)) { + throw new IllegalArgumentException("kitodoScript.importProcesses.errors.isNoDirectory"); + } + if (!Files.isWritable(errorDir)) { + throw new IllegalArgumentException("kitodoScript.importProcesses.errors.cannotWrite"); + } + return errorDir; + } + + /* + * The method is used by the Task Manager to spawn a new thread for this + * task, if the previous thread was prematurely stopped. (A thread cannot + * be continued under Java, a new thread object must be required.) + */ + @Override + public ImportProcesses replace() { + return new ImportProcesses(this); + } + + /* + * The thread's runner method. This does the real work. Each importing + * processes are always driven to individual work steps again and again, + * after which the method returns to increase the progress and allow the + * task to be stopped. This is complicated, but gives the task additional + * flexibility to control. + */ + @Override + public void run() { + try { + while (step < totalActions) { + run(step); + super.setProgress(100d * ++step / totalActions); + if (super.isInterrupted()) { + return; + } + } + // error barrier + } catch (IOException | DAOException | DataException | ProcessGenerationException | MediaNotFoundException + | InvalidImagesException | RuntimeException exception) { + Helper.setErrorMessage(exception.getLocalizedMessage(), logger, exception); + super.setException(exception); + } + } + + void run(int setStep) throws IOException, DAOException, DataException, ProcessGenerationException, + MediaNotFoundException, InvalidImagesException { + + step = setStep; + Path processesPath = Paths.get(KitodoConfig.getKitodoDataDirectory()); + if (step == 0) { + initialize(); + } else if (step <= numberOfImportingProcesses) { + validate(); + } else { + copyFilesAndCreateDatabaseEntry(step, processesPath); + } + } + + private void initialize() throws IOException { + super.setWorkDetail(importRootPath.toString()); + ruleset = ServiceManager.getRulesetManagementService().getRulesetManagement(); + ruleset.load(new File(Paths.get(ConfigCore.getParameter(ParameterCore.DIR_RULESETS), + templateForProcesses.getRuleset().getFile()).toString())); + totalActions = importingProcesses.entrySet().parallelStream().map(Entry::getValue) + .mapToInt(ImportingProcess::numberOfActions).sum() + INIT_ACTIONS_COUNT; + importingProcessesIterator = importingProcesses.values().iterator(); + } + + private void validate() throws IOException, DAOException { + validatingImportingProcess = importingProcessesIterator.next(); + super.setWorkDetail(validatingImportingProcess.directoryName); + validatingImportingProcess.validate(ruleset, strictValidation, importingProcesses); + // in last iteration, re-initialize iterator + if (step == numberOfImportingProcesses) { + /* Children must be imported before their parents, so + * that when the parents are imported, the process ID of + * the children is already known and can be written down + * in the METS file. */ + TreeSet childrenFirst = new TreeSet( + Comparator.comparingInt(ImportingProcess::childDepth).thenComparing(ImportingProcess::toString)); + childrenFirst.addAll(importingProcesses.values()); + importingProcessesIterator = childrenFirst.iterator(); + } + } + + private void copyFilesAndCreateDatabaseEntry(int step, Path processesPath) throws IOException, DAOException, DataException, + ProcessGenerationException, MediaNotFoundException, InvalidImagesException { + if (nextAction == numberOfRemainingActions && step < totalActions - 1) { + currentlyImporting = importingProcessesIterator.next(); + currentlyImporting.setProject(project); + currentlyImporting.setTemplate(templateForProcesses); + nextAction = 0; + currentlyImporting.setOutputRoot(currentlyImporting.isCorrect() ? processesPath : errorPath); + numberOfRemainingActions = currentlyImporting.numberOfActions() - VALIDATION_ACTIONS_COUNT; + } + super.setWorkDetail(currentlyImporting.directoryName); + currentlyImporting.executeAction(nextAction); + nextAction++; + } +} diff --git a/Kitodo/src/main/java/org/kitodo/production/services/command/ImportingProcess.java b/Kitodo/src/main/java/org/kitodo/production/services/command/ImportingProcess.java new file mode 100644 index 00000000000..12164defda5 --- /dev/null +++ b/Kitodo/src/main/java/org/kitodo/production/services/command/ImportingProcess.java @@ -0,0 +1,610 @@ +/* + * (c) Kitodo. Key to digital objects e. V. + * + * This file is part of the Kitodo project. + * + * It is licensed under GNU General Public License version 3 or later. + * + * For the full copyright and license information, please read the + * GPL3-License.txt file that was distributed with this source code. + */ + +package org.kitodo.production.services.command; + +// static functions used +import static java.lang.System.lineSeparator; +import static java.util.function.Predicate.not; +import static java.util.stream.Collectors.toList; + +// base Java +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +// open source code +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.kitodo.api.Metadata; +import org.kitodo.api.MetadataEntry; +import org.kitodo.api.dataeditor.rulesetmanagement.RulesetManagementInterface; +import org.kitodo.api.dataformat.LogicalDivision; +import org.kitodo.api.dataformat.Workpiece; +import org.kitodo.api.dataformat.mets.LinkedMetsResource; +import org.kitodo.api.validation.State; +import org.kitodo.api.validation.ValidationResult; +import org.kitodo.data.database.beans.Process; +import org.kitodo.data.database.beans.Project; +import org.kitodo.data.database.beans.Task; +import org.kitodo.data.database.beans.Template; +import org.kitodo.data.database.exceptions.DAOException; +import org.kitodo.data.exceptions.DataException; +import org.kitodo.exceptions.InvalidImagesException; +import org.kitodo.exceptions.MediaNotFoundException; +import org.kitodo.exceptions.ProcessGenerationException; +import org.kitodo.production.helper.Helper; +import org.kitodo.production.process.NewspaperProcessesGenerator; +import org.kitodo.production.process.ProcessGenerator; +import org.kitodo.production.services.ServiceManager; +import org.kitodo.production.services.data.ProcessService; +import org.kitodo.production.services.data.TaskService; +import org.kitodo.production.services.dataformat.MetsService; +import org.kitodo.production.services.file.FileService; +import org.kitodo.production.services.validation.MetadataValidationService; + +/** + * A process to import. For each process to be imported (that is, a process + * directory with a {@code meta.xml} file and sub-folders with media files), + * such an object is created, that imports this process into the application + * logic. The existence of the {@code meta.xml} file is checked and its content + * is validated against the specified ruleset. In the event of an error, an + * error information text file is written. Furthermore, a special treatment + * represents the handling of child processes. + */ +final class ImportingProcess { + private static final Logger logger = LogManager.getLogger(ImportingProcess.class); + + // Constants + private static final String META_FILE_NAME = "meta.xml"; + + // Services in use + private final FileService fileService = ServiceManager.getFileService(); + private final MetadataValidationService metadataValidationService = ServiceManager.getMetadataValidationService(); + private final MetsService metsService = ServiceManager.getMetsService(); + private final ProcessService processService = ServiceManager.getProcessService(); + private final TaskService taskService = ServiceManager.getTaskService(); + + /* + * This class makes extensive use of global variables (fields). This is due + * to the fact that the methods of the class do not run from front to back, + * but the sub-steps are called individually by the task manager in order to + * implement the ability to stop and restart the task. + */ + + // Input directories and files + private Path sourceDir; + final String directoryName; + private List filesAndDirectories = new ArrayList<>(); + private int numberOfFileSystemItems; + private Iterator filesAndDirectoriesIterator; + + // Process hierarchy + private Map importingProcesses; + private ImportingProcess parent; + private final List children = new ArrayList<>(); + + // Errors + private Set sharedErroneousProcesses = null; + final List errors = new ArrayList<>(); + + // database process + private Project project; + private Template template; + private String processTitleRule; + private String title; + private String baseType; + private Integer processId; + + // Output directories + private Path copyToRoot; + private Path outputDir; + + /** + * Constructor. Creates a new Importing Process. + * + * @param sourceDir + * source directory of process to import + */ + ImportingProcess(Path sourceDir) { + this.sourceDir = sourceDir; + this.directoryName = sourceDir.getFileName().toString(); + try (Stream pathStream = Files.walk(sourceDir)) { + for (Path entry : (Iterable) pathStream::iterator) { + if (!entry.equals(sourceDir)) { + filesAndDirectories.add(sourceDir.relativize(entry)); + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + numberOfFileSystemItems = filesAndDirectories.size(); + } + + /** + * Specifies the number of actions required to import this process. This is + * used to display the progress bar in Task Manager. + * + *

+ * One action is the validation. One action is to create the directory for + * the import. One action is either creating the process in the database + * or writing the error file. One action each is creating a + * sub-directory or copying a file. + * + * @return sum of actions needed + */ + int numberOfActions() { + int result = numberOfFileSystemItems + 3; + if (logger.isTraceEnabled()) { + logger.trace("Import folder {}: {} actions required. List: {}", this.directoryName, result, + filesAndDirectories.stream().map(Objects::toString).collect(Collectors.joining(", "))); + } + return result; + } + + /** + * Validates the METS file against the ruleset, generates the process title, + * and checks for the existence of all child processes listed. Validation is + * performed extensively to uncover and report as many bugs as possible in + * one run. + * + * @param ruleset + * the ruleset to validate against + * @param strictValidation + * whether warnings are an error + * @param importingProcesses + * access to all processes in this import run, to confirm the + * presence of child processes + */ + void validate(RulesetManagementInterface ruleset, boolean strictValidation, + Map importingProcesses) throws IOException, DAOException { + + this.importingProcesses = importingProcesses; + logger.info("Starting to validate " + this.directoryName); + Path metaFilePath = sourceDir.resolve(META_FILE_NAME); + Workpiece workpiece = metsService.loadWorkpiece(metaFilePath.toUri()); + validateMetsFile(ruleset, strictValidation, workpiece); + validateChildren(workpiece.getLogicalStructure()); + if (!errors.isEmpty()) { + getSharedErroneousProcesses().add(this.directoryName); + } + + // determine data for process object + baseType = workpiece.getLogicalStructure().getType(); + title = formProcessTitle(ruleset, workpiece); + + logger.info("Validation of " + this.directoryName + (errors.isEmpty() ? " completed without errors" + : " completed with errors:" + lineSeparator() + String.join(lineSeparator(), errors))); + } + + /** + * Validates the METS file against the ruleset. Possible rule violations + * will be reported. + * + * @param ruleset + * the ruleset to validate against + * @param strictValidation + * whether warnings are an error + * @param workpiece + * the workpiece to be validated + */ + private void validateMetsFile(RulesetManagementInterface ruleset, boolean strictValidation, Workpiece workpiece) + throws DAOException { + ValidationResult validationResult = metadataValidationService.validate(workpiece, ruleset, false); + State state = validationResult.getState(); + if (State.ERROR.equals(state) || (strictValidation && !State.SUCCESS.equals(state))) { + errors.add(Helper.getTranslation("dataEditor.validation.state.error").concat(":")); + for (String resultMessage : validationResult.getResultMessages()) { + errors.add(" - ".concat(resultMessage)); + } + logger.info(String.join(System.lineSeparator(), errors)); + } + } + + /** + * Checks for the presence of linked child processes. If not, then this will + * break this process, but also any existing child processes. + * + * @param logicalStructure + * in the logical structure of the process, the children are + * listed, if there are any + */ + private void validateChildren(LogicalDivision logicalStructure) { + Set linkedChildren = searchLinkedProcesses(logicalStructure).keySet(); + List problemChildren = linkedChildren.parallelStream().filter(not(importingProcesses::containsKey)) + .collect(toList()); + if (!problemChildren.isEmpty()) { + String errorMessage = Helper.getTranslation("kitodoScript.importProcesses.missingChildren", + this.directoryName, String.join(", ", problemChildren)); + this.errors.add(errorMessage); + } + for (String linkedChild : linkedChildren) { + ImportingProcess importingChild = importingProcesses.get(linkedChild); + if (Objects.nonNull(importingChild)) { + this.children.add(importingChild); + importingChild.parent = this; + } + } + } + + /** + * Recursively determines the existence of linked child processes and + * returns any found ones. + * + * @param division + * division under which the search is made + * @return found references, with name and the division under which it is + * located + */ + private Map searchLinkedProcesses(LogicalDivision division) { + Map result = new HashMap<>(); + if (division.getChildren().isEmpty()) { + if (Objects.nonNull(division.getLink())) { + String name = division.getLink().getUri().toString(); + int firstEqualsSign = name.indexOf('='); + if (firstEqualsSign > -1) { + name = name.substring(firstEqualsSign + 1); + } + result.put(name, division); + } + } else { + for (LogicalDivision child : division.getChildren()) { + result.putAll(searchLinkedProcesses(child)); + } + } + return result; + } + + /** + * Forms the process title. To do this, the formation rule is obtained from + * the ruleset, and then the composition is executed. + * + * @param ruleset + * the ruleset to validate against + * @param workpiece + * the workpiece to be validated + * @return the process title. May be {@code null} if the process title + * cannot yet be formed, because it requires a parent process that + * has not yet been initialized + */ + private String formProcessTitle(RulesetManagementInterface ruleset, Workpiece workpiece) { + Optional processTitleAttribute = ruleset + .getStructuralElementView(baseType, "create", NewspaperProcessesGenerator.ENGLISH).getProcessTitle(); + if (processTitleAttribute.isEmpty()) { + errors.add(Helper.getTranslation("kitodoScript.importProcesses.noProcessTitleRule", baseType)); + return null; + } + processTitleRule = processTitleAttribute.get(); + if (!processTitleRule.startsWith("+") || (parent != null && parent.title != null)) { + String processTitle = calculateProcessTitle(processTitleRule, parent == null ? null : parent.title, + workpiece.getLogicalStructure().getMetadata()); + processTitleRule = null; + return processTitle; + } + return null; + } + + /** + * Calculates the process title. + * + * @param rule + * formation rule + * @param parentTitle + * title of the parent process + * @param metadata + * metadata of the current process + * @return the process title + */ + private String calculateProcessTitle(String rule, String parentTitle, Collection metadata) { + List components = Arrays.asList(rule.split("\\+")); + for (int i = 0; i < components.size(); i++) { + String component = components.get(i); + if (i == 0 && component.isEmpty()) { + components.set(i, parentTitle); + } else if (component.matches("'.*'")) { + components.set(i, component.substring(1, component.length() - 1)); + } else { + Optional lookedUpValue = metadata.parallelStream().filter(MetadataEntry.class::isInstance) + .filter(metadataEntry -> metadataEntry.getKey().equals(component)) + .map(MetadataEntry.class::cast).map(MetadataEntry::getValue).findAny(); + if (lookedUpValue.isPresent()) { + components.set(i, lookedUpValue.get()); + } + } + } + return String.join("", components); + } + + /** + * Returns whether the process to import successfully validated. + * + * @return whether the process is correct + */ + boolean isCorrect() { + getSharedErroneousProcesses(); + if (!errors.isEmpty()) { + sharedErroneousProcesses.add(this.directoryName); + } + return sharedErroneousProcesses.isEmpty(); + } + + private Set getSharedErroneousProcesses() { + if (Objects.isNull(sharedErroneousProcesses)) { + setSharedErroneousProcesses( + Objects.isNull(parent) ? new HashSet<>() : parent.getSharedErroneousProcesses()); + } + return sharedErroneousProcesses; + } + + private void setSharedErroneousProcesses(Set sharedErroneousProcesses) { + this.sharedErroneousProcesses = sharedErroneousProcesses; + for (ImportingProcess importingChild : children) { + importingChild.setSharedErroneousProcesses(sharedErroneousProcesses); + } + } + + // === PROCESSING === + + /** + * Creates the directory where the process should be copied to. It can be + * {@code null} in case of a validation error, then the faulty process is + * not copied anywhere. + * + * @param copyToRoot + * copy target + */ + void setOutputRoot(Path copyToRoot) throws IOException { + this.copyToRoot = copyToRoot; + } + + /** + * Executes a processing step. The method is called from the Task Manager + * once per processing step. Because validation is called separately, the + * number of calls here is one less than the number returned by + * {@link #numberOfActions()}. In other words, {@code action} must be in the + * range 0 to {@code (numberOfActions() - 2)}. + * + * @param action + * processing step + * @throws IOException + * if accessing the file system fails + * @throws DAOException + * if accessing the database fails + * @throws DataException + * if accessing the database fails + * @throws ProcessGenerationException + * if creating the process fails + * @throws InvalidImagesException + * if media file names in the METS file cannot be aligned with + * the project configuration + * @throws MediaNotFoundException + * if media files are missing + */ + void executeAction(int action) throws IOException, DAOException, DataException, ProcessGenerationException, + MediaNotFoundException, InvalidImagesException { + + assert action >= 0 && action <= numberOfFileSystemItems + 1 + : "action out of range: " + action + " [0.." + (numberOfFileSystemItems + 1) + "]"; + if (isCorrect()) { + executeActionForCorrectProcess(action); + } else if (Objects.nonNull(copyToRoot)) { + executeActionForErroreousProcess(action); + } + } + + /** + * Performs a processing step on a successfully validated process. First, + * the process is created in the database to get its ID, which is the name + * of the process directory. In the second step, this directory is created. + * Then all files will be copied into it, except for the {@code meta.xml}, + * which will be skipped. The {@code meta.xml} is transferred at the end and + * adjusted while doing so. + * + * @param action + * processing step + */ + private void executeActionForCorrectProcess(int action) throws IOException, DataException, + ProcessGenerationException, DAOException, InvalidImagesException, MediaNotFoundException { + + if (action == 0) { + if (Objects.nonNull(processTitleRule) && Objects.isNull(title)) { + Workpiece workpiece = metsService.loadWorkpiece(sourceDir.resolve(META_FILE_NAME).toUri()); + title = calculateProcessTitle(processTitleRule, parent.title, + workpiece.getLogicalStructure().getMetadata()); + } + processId = createDatabaseProcess(); + logger.info("Created process #" + processId); + } else if (action == 1) { + createBaseDirectory(processId.toString()); + filesAndDirectoriesIterator = filesAndDirectories.iterator(); + } else if (filesAndDirectoriesIterator.hasNext()) { + Path relativeItemToCopy = filesAndDirectoriesIterator.next(); + if (relativeItemToCopy.toString().equals(META_FILE_NAME)) { + if (filesAndDirectoriesIterator.hasNext()) { + relativeItemToCopy = filesAndDirectoriesIterator.next(); + } else { + Process process = processService.getById(processId); + copyAndAdjustMetsFile(process); + processService.save(process); + } + } else { + copyDirectoryOrFile(relativeItemToCopy); + } + } else { + Process process = processService.getById(processId); + copyAndAdjustMetsFile(process); + processService.save(process); + } + } + + /** + * Creates a process in the database. The process title must have already + * been generated during validation. + * + * @return database number of the created process + */ + private Integer createDatabaseProcess() throws ProcessGenerationException, DataException { + ProcessGenerator processGenerator = new ProcessGenerator(); + processGenerator.generateProcess(template.getId(), project.getId()); + Process process = processGenerator.getGeneratedProcess(); + process.setTitle(title); + process.setBaseType(baseType); + processService.save(process); + for (Task task : process.getTasks()) { + taskService.save(task); + } + return process.getId(); + } + + /** + * Transfers and adapts the {@code meta.xml}. The new process number is + * stored, the links are updated with the new IDs of the imported children, + * and media files are added. + * + * @param process + * the newly created process + */ + private void copyAndAdjustMetsFile(Process process) + throws IOException, InvalidImagesException, MediaNotFoundException, DAOException, DataException { + + Workpiece workpiece = metsService.loadWorkpiece(sourceDir.resolve(META_FILE_NAME).toUri()); + workpiece.setId(processId.toString()); + Map linkedChildren = searchLinkedProcesses(workpiece.getLogicalStructure()); + for (Entry linkedChild : linkedChildren.entrySet()) { + LinkedMetsResource link = linkedChild.getValue().getLink(); + link.setLoctype("Kitodo.Production"); + ImportingProcess importedChildProcess = importingProcesses.get(linkedChild.getKey()); + link.setUri(processService.getProcessURI(importedChildProcess.processId)); + addLinkInDatabase(process, importedChildProcess.processId); + } + fileService.searchForMedia(process, workpiece); + Path outputMetsFile = outputDir.resolve(META_FILE_NAME); + metsService.saveWorkpiece(workpiece, outputMetsFile.toUri()); + logger.info("Wrote METS file " + outputMetsFile); + } + + private void addLinkInDatabase(Process parent, Integer childProcessId) throws DAOException, DataException { + Process child = processService.getById(childProcessId); + child.setParent(parent); + parent.getChildren().add(child); + processService.save(child); + } + + /** + * Processes a faulty process. The output folder will be created in the + * error target folder. The fault information file is written in it. And + * then all the content is copied over. + * + * @param action + * action to take. The processing does not take place in one run, + * but the function is called by the task thread once per step. + * This is necessary to ensure stopability and the progress + * display in the task manager. + */ + private void executeActionForErroreousProcess(int action) throws IOException { + if (action == 0) { + createBaseDirectory(this.directoryName); + } else if (action == 1) { + String nameOfErrorFile = Helper.getTranslation("errors").concat(".txt"); + Path errorFile = outputDir.resolve(nameOfErrorFile); + if (errors.isEmpty()) { + String message = Helper.getTranslation("kitodoScript.importProcesses.brokenRelatedProcess", + String.join(", ", sharedErroneousProcesses)); + Files.writeString(errorFile, message, StandardCharsets.UTF_8); + } else { + Files.write(errorFile, errors, StandardCharsets.UTF_8); + } + logger.info("Wrote errors file " + errorFile); + } else { + copyDirectoryOrFile(filesAndDirectories.get(action - 2)); + } + } + + /** + * Determines the number of levels of children below this process. + * + * @return number of levels of children below + */ + int childDepth() { + OptionalInt childrenMaxDepth = children.parallelStream().mapToInt(ImportingProcess::childDepth).max(); + if (childrenMaxDepth.isEmpty()) { + return 0; + } else { + return childrenMaxDepth.getAsInt() + 1; + } + } + + // === UTILITIES === + + /** + * Creates the directory for the output. + * + * @param directoryName + * the name for the directory + */ + private void createBaseDirectory(String directoryName) throws IOException { + outputDir = copyToRoot.resolve(directoryName); + Files.createDirectories(outputDir); + logger.info("Created process directory " + outputDir); + } + + /** + * Copies a directory (without content) or a file from the source directory + * to the output directory. + * + * @param relativeItem + * the relative path, relative to the respective root + */ + private void copyDirectoryOrFile(Path relativeItem) throws IOException { + Path sourceItem = sourceDir.resolve(relativeItem); + Path destinationItem = outputDir.resolve(relativeItem); + if (Files.isDirectory(sourceItem)) { + Files.createDirectories(destinationItem); + logger.info("Created directory " + destinationItem); + } else { + Files.copy(sourceItem, destinationItem, StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.COPY_ATTRIBUTES, LinkOption.NOFOLLOW_LINKS); + logger.info("Copied " + sourceItem + " as " + destinationItem); + } + } + + void setProject(Project project) { + this.project = project; + } + + void setTemplate(Template template) { + this.template = template; + } + + @Override + public String toString() { + return "Importing process " + sourceDir; + } +} diff --git a/Kitodo/src/main/java/org/kitodo/production/services/command/KitodoScriptService.java b/Kitodo/src/main/java/org/kitodo/production/services/command/KitodoScriptService.java index ae2a73688f5..ac4abe05ba1 100644 --- a/Kitodo/src/main/java/org/kitodo/production/services/command/KitodoScriptService.java +++ b/Kitodo/src/main/java/org/kitodo/production/services/command/KitodoScriptService.java @@ -229,6 +229,18 @@ private boolean executeRemainingScript(List processes, String script) case "searchForMedia": searchForMedia(processes); break; + case "importProcesses": + String indir = parameters.get("indir"); + String project = parameters.get("project"); + String template = parameters.get("template"); + String errors = parameters.get("errors"); + try { + TaskManager.addTask(new ImportProcesses(indir, project, template, errors)); + Helper.setMessage("kitodoScript.importProcesses.executesInTaskManager"); + } catch (IllegalArgumentException e) { + Helper.setErrorMessage(e.getMessage()); + } + break; default: Helper.setErrorMessage("Unknown action", " - use: 'action:addRole, action:setTaskProperty, action:setStepStatus, " diff --git a/Kitodo/src/main/java/org/kitodo/production/services/validation/MetadataValidationService.java b/Kitodo/src/main/java/org/kitodo/production/services/validation/MetadataValidationService.java index e27a83060b4..a8014978b60 100644 --- a/Kitodo/src/main/java/org/kitodo/production/services/validation/MetadataValidationService.java +++ b/Kitodo/src/main/java/org/kitodo/production/services/validation/MetadataValidationService.java @@ -170,16 +170,36 @@ public ValidationResult validate(URI metsFileUri, URI rulesetFileUri) { * @param ruleset * Ruleset file * @return the validation result - * @throws DataException - * if an error occurs while reading from the search engine + * @throws DAOException + * if an error occurs while reading from the database */ public ValidationResult validate(Workpiece workpiece, RulesetManagementInterface ruleset) throws DAOException { + return validate(workpiece, ruleset, true); + } + + /** + * Validates a workpiece based on a rule set. + * + * @param workpiece + * METS file + * @param ruleset + * Ruleset file + * @param strict + * whether to validate document ID and presence of images + * @return the validation result + * @throws DAOException + * if an error occurs while reading from the database + */ + public ValidationResult validate(Workpiece workpiece, RulesetManagementInterface ruleset, boolean strict) + throws DAOException { Collection results = new ArrayList<>(); - results.add(checkTheIdentifier(workpiece)); + if (strict) { + results.add(checkTheIdentifier(workpiece)); + } results.add(metadataValidation.validate(workpiece, ruleset, getMetadataLanguage(), - getTranslations())); + getTranslations(), strict)); return merge(results); } @@ -189,8 +209,8 @@ public ValidationResult validate(Workpiece workpiece, RulesetManagementInterface * @param workpiece * METS file * @return the validation result - * @throws DataException - * if an error occurs while reading from the search engine + * @throws DAOException + * if an error occurs while reading from the database */ private ValidationResult checkTheIdentifier(Workpiece workpiece) throws DAOException { boolean error = false; diff --git a/Kitodo/src/main/resources/messages/errors_de.properties b/Kitodo/src/main/resources/messages/errors_de.properties index 359887cf370..7f798275246 100644 --- a/Kitodo/src/main/resources/messages/errors_de.properties +++ b/Kitodo/src/main/resources/messages/errors_de.properties @@ -10,6 +10,7 @@ # error=Fehler... +errors=Fehler # A errorAdding=Fehler beim Hinzuf\u00FCgen von ''{0}'' @@ -88,6 +89,21 @@ invalidIdentifierSame=Das Identifikationsmerkmal {0} enth\u00E4lt in {1} und {2} # J # K +kitodoScript.importProcesses.brokenRelatedProcess=Vorgang wurde wegen Fehlern in verkn\u00FCpften Vorg\u00E4ngen {0} nicht importiert. +kitodoScript.importProcesses.errors.cannotWrite=Fehlende Schreibberechtigung im Verzeichniss ''errors''! +kitodoScript.importProcesses.errors.isNoDirectory=''errors'' ist kein existierendes Verzeichnis! +kitodoScript.importProcesses.executesInTaskManager=Das KitodoSkript wird im Task Manager ausgef\u00FChrt. +kitodoScript.importProcesses.indir.isNull=Parameter ''indir'' fehlt! +kitodoScript.importProcesses.indir.isNoDirectory=''indir'' ist kein existierendes Verzeichnis! +kitodoScript.importProcesses.indir.cannotExecute=Der Inhalt des Importverzeichnisses kann nicht abgerufen werden (fehlende Berechtigung \u201Cexecute\u201D) +kitodoScript.importProcesses.missingChildren=Vorgang {0} enth\u004E Verweise auf fehlendene Kindvorg\u004Enge {1} +kitodoScript.importProcesses.noProcessTitleRule=Regelsatz gibt keine Bildungsvorschrift für den ''processTitle'' für Division {0} an! +kitodoScript.importProcesses.project.isNull=Parameter ''project'' fehlt! +kitodoScript.importProcesses.project.isNoProjectID=''project'' ist keine g\u00FCltige Projekt-ID +kitodoScript.importProcesses.project.noProjectWithID=Ein solches Projekt existiert nicht! +kitodoScript.importProcesses.template.isNull=Parameter ''template'' fehlt! +kitodoScript.importProcesses.template.isNoTemplateID=''template'' ist keine g\u00F6ltige Vorlagen-ID +kitodoScript.importProcesses.template.noTemplateWithID=Eine solche Produktionsvorlage existiert nicht! # L errorLoadingDocTypes=Regelsatz Konfigurationsfehler: Kein DocType ('division') gefunden diff --git a/Kitodo/src/main/resources/messages/errors_en.properties b/Kitodo/src/main/resources/messages/errors_en.properties index 9ed5f8fc905..2929d7d7366 100644 --- a/Kitodo/src/main/resources/messages/errors_en.properties +++ b/Kitodo/src/main/resources/messages/errors_en.properties @@ -10,6 +10,7 @@ # error=Error... +errors=Errors # A errorAdding=Error adding ''{0}'' @@ -88,6 +89,21 @@ invalidIdentifierSame=The identifier {0} has the same value in {1} and {2}. # J # K +kitodoScript.importProcesses.brokenRelatedProcess=Process was not imported due to errors in related process(es): {0} +kitodoScript.importProcesses.errors.cannotWrite=Cannot write in location ''errors''! +kitodoScript.importProcesses.errors.isNoDirectory=''errors'' is not a valid directory! +kitodoScript.importProcesses.executesInTaskManager=Executing Kitodo Script in the Task Manager. +kitodoScript.importProcesses.indir.isNull=Missing value for ''indir''! +kitodoScript.importProcesses.indir.isNoDirectory=''indir'' is not a valid directory! +kitodoScript.importProcesses.indir.cannotExecute=Cannot list ''indir'' directory (missing execute permission) +kitodoScript.importProcesses.missingChildren=Process {0} contains references to missing child processes {1} +kitodoScript.importProcesses.noProcessTitleRule=Ruleset does not specify a ''processTitle'' rule for division {0} +kitodoScript.importProcesses.project.isNull=Missing value for ''project''! +kitodoScript.importProcesses.project.isNoProjectID=''project'' is not int! +kitodoScript.importProcesses.project.noProjectWithID=There is no project with that ID +kitodoScript.importProcesses.template.isNull=Missing value for ''template''! +kitodoScript.importProcesses.template.isNoTemplateID=''template'' is not int! +kitodoScript.importProcesses.template.noTemplateWithID=There is no production template with that ID # L errorLoadingDocTypes=Ruleset configuration error: No DocType ('division') found diff --git a/Kitodo/src/main/resources/messages/errors_es.properties b/Kitodo/src/main/resources/messages/errors_es.properties index 605c55f6580..ad58615820e 100644 --- a/Kitodo/src/main/resources/messages/errors_es.properties +++ b/Kitodo/src/main/resources/messages/errors_es.properties @@ -10,6 +10,7 @@ # error=error... +errors=Errores # A errorAdding=Error al añadir ''{0}'' @@ -87,6 +88,21 @@ invalidIdentifierSame=La característica de identificación {0} contiene el mism # J # K +kitodoScript.importProcesses.brokenRelatedProcess=Proceso no se agrega debido a errores en los procesos relaciónados {0}. +kitodoScript.importProcesses.errors.isNoDirectory=¡''errors'' no es una lista de archivos! +kitodoScript.importProcesses.errors.cannotWrite=¡No puedo describir el lugar ''errors''! +kitodoScript.importProcesses.executesInTaskManager=Voy a llevar al guión de Kitodo al administrador de tareas pendientes. +kitodoScript.importProcesses.indir.isNull=¡Falta la variable ''indir''! +kitodoScript.importProcesses.indir.isNoDirectory=¡''indir'' no es una lista de archivos! +kitodoScript.importProcesses.indir.cannotExecute=No se puede obtener la lista del directorio ''indir'' (falta de permiso para ejecutar) +kitodoScript.importProcesses.missingChildren=El proceso {0} contiene referencias al proceso {1} del niño perdido. +kitodoScript.importProcesses.noProcessTitleRule=Declaración de reglas no especifica una regla de de educación ''processTitle'' para la division {0} +kitodoScript.importProcesses.project.isNull=¡Falta la variable ''project''! +kitodoScript.importProcesses.project.isNoProjectID=''project'' no es un ID válido de proyecto (debe ser entero) +kitodoScript.importProcesses.project.noProjectWithID=¡Tal proyecto no existe! +kitodoScript.importProcesses.template.isNull=¡Falta la variable ''template''! +kitodoScript.importProcesses.template.isNoTemplateID=¡''template'' no es un ID válido en la plantilla! +kitodoScript.importProcesses.template.noTemplateWithID=No existe tal plantilla con estos dígitos de identificación ''template'' # L errorLoadingDocTypes=Error de configuración del conjunto de reglas: No se ha encontrado ningún DocType ('division') diff --git a/Kitodo/src/main/resources/messages/messages_de.properties b/Kitodo/src/main/resources/messages/messages_de.properties index 4e8d28a0369..977d2061390 100644 --- a/Kitodo/src/main/resources/messages/messages_de.properties +++ b/Kitodo/src/main/resources/messages/messages_de.properties @@ -645,6 +645,7 @@ importingData=Daten werden importiert... importChildren=Kindvorg\u00E4nge importieren importDepth=Importtiefe importDms=Export in das DMS +importProcesses=Vorg\u00E4nge importieren imprint=Impressum # used in "LegalTexts.java" imprintDefaultText=Das Impressum f\u00FCr dieses System wurde noch nicht hinterlegt. diff --git a/Kitodo/src/main/resources/messages/messages_en.properties b/Kitodo/src/main/resources/messages/messages_en.properties index c38c291cedc..d95a8ed8ffd 100644 --- a/Kitodo/src/main/resources/messages/messages_en.properties +++ b/Kitodo/src/main/resources/messages/messages_en.properties @@ -646,6 +646,7 @@ importingData=Data are being imported importChildren=Import child processes importDepth=Import depth importDms=Import into DMS +importProcesses=Import processes imprint=Imprint # used in "LegalTexts.java" imprintDefaultText=Imprint information have not been configured for this system. diff --git a/Kitodo/src/main/resources/messages/messages_es.properties b/Kitodo/src/main/resources/messages/messages_es.properties index 6d8b2c01fcd..0effd29d4bc 100644 --- a/Kitodo/src/main/resources/messages/messages_es.properties +++ b/Kitodo/src/main/resources/messages/messages_es.properties @@ -639,6 +639,7 @@ importingData=Los datos se importan... importChildren=Importar procesos hijos importDepth=Profundidad de importación importDms=Exportación a DMS +importProcesses=Importacione de procesos imprint=Pie de imprenta # used in "LegalTexts.java" imprintDefaultText=La huella de este sistema aún no se ha almacenado. diff --git a/Kitodo/src/main/webapp/WEB-INF/templates/includes/processes/executeScriptSelectedPopup.xhtml b/Kitodo/src/main/webapp/WEB-INF/templates/includes/processes/executeScriptSelectedPopup.xhtml index 1207f4ad990..9f82edc96a0 100644 --- a/Kitodo/src/main/webapp/WEB-INF/templates/includes/processes/executeScriptSelectedPopup.xhtml +++ b/Kitodo/src/main/webapp/WEB-INF/templates/includes/processes/executeScriptSelectedPopup.xhtml @@ -89,6 +89,9 @@ +

diff --git a/Kitodo/src/test/java/org/kitodo/production/services/command/ImportProcessesIT.java b/Kitodo/src/test/java/org/kitodo/production/services/command/ImportProcessesIT.java new file mode 100644 index 00000000000..30630c31157 --- /dev/null +++ b/Kitodo/src/test/java/org/kitodo/production/services/command/ImportProcessesIT.java @@ -0,0 +1,332 @@ +/* + * (c) Kitodo. Key to digital objects e. V. + * + * This file is part of the Kitodo project. + * + * It is licensed under GNU General Public License version 3 or later. + * + * For the full copyright and license information, please read the + * GPL3-License.txt file that was distributed with this source code. + */ + +package org.kitodo.production.services.command; + +// abbreviations +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +// base Java +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Date; +import java.time.LocalDate; +import java.time.ZoneId; + +// open source code +import org.apache.commons.lang3.SystemUtils; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.kitodo.ExecutionPermission; +import org.kitodo.MockDatabase; +import org.kitodo.SecurityTestUtils; +import org.kitodo.TreeDeleter; +import org.kitodo.config.ConfigCore; +import org.kitodo.config.enums.ParameterCore; +import org.kitodo.data.database.beans.Client; +import org.kitodo.data.database.beans.Folder; +import org.kitodo.data.database.beans.Process; +import org.kitodo.data.database.beans.Ruleset; +import org.kitodo.data.database.beans.Task; +import org.kitodo.data.database.beans.Template; +import org.kitodo.data.database.beans.User; +import org.kitodo.production.services.ServiceManager; +import org.kitodo.test.utils.ProcessTestUtils; + +public class ImportProcessesIT { + private static final Path ERRORS_DIR_PATH = Paths.get("src/test/resources/errors"); + + private int firstProcessId, secondProcessId, thirdProcessId; + private static Template template; + + @BeforeClass + public static void prepareDatabase() throws Exception { + MockDatabase.startNode(); + MockDatabase.insertProcessesFull(); + + // add ruleset + Ruleset ruleset = new Ruleset(); + ruleset.setTitle("ImportProcessesIT"); + ruleset.setFile("ImportProcessesIT.xml"); + ruleset.setOrderMetadataByRuleset(true); + Client clientOne = ServiceManager.getClientService().getById(1); + ruleset.setClient(clientOne); + ServiceManager.getRulesetService().save(ruleset); + + Task task = new Task(); + task.getRoles().add(ServiceManager.getRoleService().getById(1)); + ServiceManager.getTaskService().save(task); + + template = new Template(); + template.setTitle("Import processes template"); + LocalDate localDate = LocalDate.of(2023, 8, 9); + template.setCreationDate(Date.from(localDate.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant())); + template.setClient(clientOne); + template.setDocket(ServiceManager.getDocketService().getById(1)); + template.getProjects().add(ServiceManager.getProjectService().getById(1)); + template.setRuleset(ruleset); + ServiceManager.getTemplateService().save(template, true); + + task.setTemplate(template); + template.getTasks().add(task); + ServiceManager.getTemplateService().save(template, true); + ServiceManager.getTaskService().save(task); + + Folder local = ServiceManager.getFolderService().getById(6); + local.setMimeType("image/jpeg"); + ServiceManager.getFolderService().saveToDatabase(local); + + User userOne = ServiceManager.getUserService().getById(1); + SecurityTestUtils.addUserDataToSecurityContext(userOne, 1); + } + + @BeforeClass + public static void setScriptPermission() throws Exception { + if (!SystemUtils.IS_OS_WINDOWS) { + ExecutionPermission + .setExecutePermission(new File(ConfigCore.getParameter(ParameterCore.SCRIPT_CREATE_DIR_META))); + } + } + + @Before + public void createOutputDirectories() throws Exception { + Files.createDirectories(ERRORS_DIR_PATH); + } + + /** + * Tests the target behavior specified. + */ + @Test + public void shouldImport() throws Exception { + // create test object + String indir = "src/test/resources/ImportProcessesIT"; + String projectId = "1"; + String templateId = template.getId().toString(); + String errors = "src/test/resources/errors"; + ImportProcesses underTest = new ImportProcesses(indir, projectId, templateId, errors); + + int processesBefore = ServiceManager.getProcessService().countDatabaseRows().intValue(); + + // initialize + underTest.run(0); + assertEquals("should require 43 actions", 43, underTest.totalActions); + + // validate: correct processes + underTest.run(1); + assertEquals("should have validated p1_valid", "p1_valid", underTest.validatingImportingProcess.directoryName); + ImportingProcess p1_valid = underTest.validatingImportingProcess; + assertTrue("should have validated without errors", p1_valid.errors.isEmpty()); + + underTest.run(2); + assertEquals("should have validated p1c1_valid", "p1c1_valid", + underTest.validatingImportingProcess.directoryName); + ImportingProcess p1c1_valid = underTest.validatingImportingProcess; + assertTrue("should have validated without errors", p1c1_valid.errors.isEmpty()); + + underTest.run(3); + assertEquals("should have validated p1c2_valid", "p1c2_valid", + underTest.validatingImportingProcess.directoryName); + ImportingProcess p1c2_valid = underTest.validatingImportingProcess; + assertTrue("should have validated without errors", p1c2_valid.errors.isEmpty()); + + assertTrue("p1_valid should be correct", p1_valid.isCorrect()); + assertTrue("p1c1_valid should be correct", p1c1_valid.isCorrect()); + assertTrue("p1c2_valid should be correct", p1c2_valid.isCorrect()); + + // validate: error case 'parent is missing a child' + underTest.run(4); + assertEquals("should have validated p2_parentMissingAChild", "p2_parentMissingAChild", + underTest.validatingImportingProcess.directoryName); + ImportingProcess p2_parentMissingAChild = underTest.validatingImportingProcess; + assertFalse("should have validated with error", p2_parentMissingAChild.errors.isEmpty()); + + underTest.run(5); + assertEquals("should have validated p2c1_valid", "p2c1_valid", + underTest.validatingImportingProcess.directoryName); + ImportingProcess p2c1_valid = underTest.validatingImportingProcess; + assertTrue("should have validated without errors", p2c1_valid.errors.isEmpty()); + + assertFalse("p1_valid should not be correct", p2_parentMissingAChild.isCorrect()); + assertFalse("p1c1_valid should not be correct, due to problem in parent case", p2c1_valid.isCorrect()); + + // validate: error case 'parent not valid' + underTest.run(6); + assertEquals("should have validated p3_not-valid", "p3_not-valid", + underTest.validatingImportingProcess.directoryName); + ImportingProcess p3_notValid = underTest.validatingImportingProcess; + assertFalse("should have validated with error", p3_notValid.errors.isEmpty()); + + underTest.run(7); + assertEquals("should have validated p3c1_valid", "p3c1_valid", + underTest.validatingImportingProcess.directoryName); + ImportingProcess p3c1_valid = underTest.validatingImportingProcess; + assertTrue("should have validated without errors", p3c1_valid.errors.isEmpty()); + + assertFalse("p1_valid should not be correct", p3_notValid.isCorrect()); + assertFalse("p1c1_valid should not be correct, due to problem in parent case", p3c1_valid.isCorrect()); + + // validate: error case 'child not valid' + underTest.run(8); + assertEquals("should have validated p4_valid_butChildIsNot", "p4_valid_butChildIsNot", + underTest.validatingImportingProcess.directoryName); + ImportingProcess p4_valid_butChildIsNot = underTest.validatingImportingProcess; + assertTrue("should have validated without errors", p4_valid_butChildIsNot.errors.isEmpty()); + + underTest.run(9); + assertEquals("should have validated p4c1_not-valid", "p4c1_not-valid", + underTest.validatingImportingProcess.directoryName); + ImportingProcess p4c1_notValid = underTest.validatingImportingProcess; + assertFalse("should have validated with error", p4c1_notValid.errors.isEmpty()); + + assertFalse("p1_valid should not be correct, due to problem in child case", p4_valid_butChildIsNot.isCorrect()); + assertFalse("p1c1_valid should not be correct", p4c1_notValid.isCorrect()); + + // copy files and create database entry + + // p1c1_valid (OK) + firstProcessId = processesBefore + 1; + Path processPath = Paths.get("src/test/resources/metadata", Integer.toString(firstProcessId)); + Path imagesPath = processPath.resolve("images"); + Path mediaPath = imagesPath.resolve("17_123_0001_media"); + Path imageOne = mediaPath.resolve("00000001.jpg"); + Path metaXml = processPath.resolve("meta.xml"); + + underTest.run(10); + assertEquals("should have created 1st process,", Long.valueOf(firstProcessId), + ServiceManager.getProcessService().countDatabaseRows()); + underTest.run(11); + assertTrue("should have created process directory", Files.isDirectory(processPath)); + underTest.run(12); + underTest.run(13); + underTest.run(14); + underTest.run(15); + + assertTrue("should have created images directory", Files.isDirectory(imagesPath)); + assertTrue("should have created media directory", Files.isDirectory(mediaPath)); + assertTrue("should have copied media file", Files.exists(imageOne)); + assertTrue("should have written meta.xml file", Files.exists(metaXml)); + assertTrue("should have added image to meta.xml file", + Files.readString(metaXml, UTF_8).contains("xlink:href=\"images/17_123_0001_media/00000001.jpg\"")); + + // p1c2_valid (OK) + secondProcessId = processesBefore + 2; + processPath = Paths.get("src/test/resources/metadata", Integer.toString(secondProcessId)); + metaXml = processPath.resolve("meta.xml"); + + underTest.run(16); + underTest.run(17); + underTest.run(18); + underTest.run(19); + underTest.run(20); + underTest.run(21); + assertTrue("should have added image to meta.xml file", + Files.readString(metaXml, UTF_8).contains("xlink:href=\"images/17_123_0002_media/00000001.jpg\"")); + + // p2c1_valid (error, broken parent) + underTest.run(22); + assertTrue("should have error directory", Files.isDirectory(Paths.get("src/test/resources/errors/p2c1_valid"))); + underTest.run(23); + assertTrue("should have written Errors.txt file", + Files.exists(Paths.get("src/test/resources/errors/p2c1_valid/Errors.txt"))); + assertTrue("should have written error message", + Files.readString(Paths.get("src/test/resources/errors/p2c1_valid/Errors.txt"), UTF_8) + .contains("errors in related process(es): p2_parentMissingAChild")); + underTest.run(24); + assertTrue("should have copied meta.xml file", + Files.exists(Paths.get("src/test/resources/errors/p2c1_valid/meta.xml"))); + + // p3c1_valid (error, broken parent) + underTest.run(25); + underTest.run(26); + assertTrue("should have written error message", + Files.readString(Paths.get("src/test/resources/errors/p3c1_valid/Errors.txt"), UTF_8) + .contains("errors in related process(es): p3_not-valid")); + underTest.run(27); + assertTrue("should have copied meta.xml file", + Files.exists(Paths.get("src/test/resources/errors/p3c1_valid/meta.xml"))); + + // p4c1_not-valid (error, not valid) + underTest.run(28); + underTest.run(29); + assertTrue("should have written error message", + Files.readString(Paths.get("src/test/resources/errors/p4c1_not-valid/Errors.txt"), UTF_8) + .contains("Validation error")); + underTest.run(30); + + // p1_valid (OK) + thirdProcessId = processesBefore + 3; + processPath = Paths.get("src/test/resources/metadata", Integer.toString(thirdProcessId)); + metaXml = processPath.resolve("meta.xml"); + + underTest.run(31); + assertEquals("should have created 3rd process,", processesBefore + 3, + (long) ServiceManager.getProcessService().countDatabaseRows()); + underTest.run(32); + assertTrue("should have created process directory", Files.isDirectory(processPath)); + underTest.run(33); + String thirdMetaXml = Files.readString(metaXml, UTF_8); + assertThat("should have added correct child links to meta.xml file", thirdMetaXml, + containsString("xlink:href=\"database://?process.id=" + firstProcessId + "\"")); + assertThat("should have added correct child links to meta.xml file", thirdMetaXml, + containsString("xlink:href=\"database://?process.id=" + secondProcessId + "\"")); + + Process parent = ServiceManager.getProcessService().getById(6); + assertEquals("parent should have 2 children", 2, parent.getChildren().size()); + assertEquals("child (ID " + firstProcessId + ") should have the correct parent", parent, + ServiceManager.getProcessService().getById(firstProcessId).getParent()); + assertEquals("child (ID " + secondProcessId + ") should have the correct parent", parent, + ServiceManager.getProcessService().getById(secondProcessId).getParent()); + + // p2_parentMissingAChild (error) + underTest.run(34); + underTest.run(35); + underTest.run(36); + + // p3_not-valid (error) + underTest.run(37); + underTest.run(38); + underTest.run(39); + + // p4_valid_butChildIsNot (error) + underTest.run(40); + underTest.run(41); + underTest.run(42); + + // import results + assertEquals("Should import 3 processes,", processesBefore + 3, + (long) ServiceManager.getProcessService().countDatabaseRows()); + assertEquals("Should not import 6 processes,", 6, ERRORS_DIR_PATH.toFile().list().length); + } + + @After + public void deleteCreatedFiles() throws Exception { + ProcessTestUtils.removeTestProcess(firstProcessId); + ProcessTestUtils.removeTestProcess(secondProcessId); + ProcessTestUtils.removeTestProcess(thirdProcessId); + TreeDeleter.deltree(ERRORS_DIR_PATH); + } + + @AfterClass + public static void cleanDatabase() throws Exception { + MockDatabase.stopNode(); + MockDatabase.cleanDatabase(); + } +} diff --git a/Kitodo/src/test/resources/ImportProcessesIT/p1_valid/meta.xml b/Kitodo/src/test/resources/ImportProcessesIT/p1_valid/meta.xml new file mode 100644 index 00000000000..a597246e01d --- /dev/null +++ b/Kitodo/src/test/resources/ImportProcessesIT/p1_valid/meta.xml @@ -0,0 +1,20 @@ + + + + +17 +123 +Lorem ipsum dolor sit amet + + Molenstraat + 21 + B + + + +
+ +
+
+ +
 diff --git a/Kitodo/src/test/resources/ImportProcessesIT/p1c1_valid/images/17_123_0001_media/00000001.jpg b/Kitodo/src/test/resources/ImportProcessesIT/p1c1_valid/images/17_123_0001_media/00000001.jpg new file mode 100644 index 00000000000..6becb4c508f Binary files /dev/null and b/Kitodo/src/test/resources/ImportProcessesIT/p1c1_valid/images/17_123_0001_media/00000001.jpg differ diff --git a/Kitodo/src/test/resources/ImportProcessesIT/p1c1_valid/meta.xml b/Kitodo/src/test/resources/ImportProcessesIT/p1c1_valid/meta.xml new file mode 100644 index 00000000000..4d1b57e258f --- /dev/null +++ b/Kitodo/src/test/resources/ImportProcessesIT/p1c1_valid/meta.xml @@ -0,0 +1,12 @@ + + + + +0001 +Lorem ipsum dolor sit amet +07.08.2023 13:31 + + + +
+ diff --git a/Kitodo/src/test/resources/ImportProcessesIT/p1c2_valid/images/17_123_0002_media/00000001.jpg b/Kitodo/src/test/resources/ImportProcessesIT/p1c2_valid/images/17_123_0002_media/00000001.jpg new file mode 100644 index 00000000000..6becb4c508f Binary files /dev/null and b/Kitodo/src/test/resources/ImportProcessesIT/p1c2_valid/images/17_123_0002_media/00000001.jpg differ diff --git a/Kitodo/src/test/resources/ImportProcessesIT/p1c2_valid/meta.xml b/Kitodo/src/test/resources/ImportProcessesIT/p1c2_valid/meta.xml new file mode 100644 index 00000000000..4cc7eaf827f --- /dev/null +++ b/Kitodo/src/test/resources/ImportProcessesIT/p1c2_valid/meta.xml @@ -0,0 +1,12 @@ + + + + +0002 +Lorem ipsum dolor sit amet +07.08.2023 13:35 + + + +
+ diff --git a/Kitodo/src/test/resources/ImportProcessesIT/p2_parentMissingAChild/meta.xml b/Kitodo/src/test/resources/ImportProcessesIT/p2_parentMissingAChild/meta.xml new file mode 100644 index 00000000000..c803d920917 --- /dev/null +++ b/Kitodo/src/test/resources/ImportProcessesIT/p2_parentMissingAChild/meta.xml @@ -0,0 +1,20 @@ + + + + +17 +123 +Lorem ipsum dolor sit amet + + Molenstraat + 21 + B + + + +
+ +
+
+ +
 diff --git a/Kitodo/src/test/resources/ImportProcessesIT/p2c1_valid/meta.xml b/Kitodo/src/test/resources/ImportProcessesIT/p2c1_valid/meta.xml new file mode 100644 index 00000000000..4cc7eaf827f --- /dev/null +++ b/Kitodo/src/test/resources/ImportProcessesIT/p2c1_valid/meta.xml @@ -0,0 +1,12 @@ + + + + +0002 +Lorem ipsum dolor sit amet +07.08.2023 13:35 + + + +
+ diff --git a/Kitodo/src/test/resources/ImportProcessesIT/p3_not-valid/meta.xml b/Kitodo/src/test/resources/ImportProcessesIT/p3_not-valid/meta.xml new file mode 100644 index 00000000000..b5317e24b8d --- /dev/null +++ b/Kitodo/src/test/resources/ImportProcessesIT/p3_not-valid/meta.xml @@ -0,0 +1,14 @@ + + + + +47 +11 +Bar + + +
+ +
+ +
 diff --git a/Kitodo/src/test/resources/ImportProcessesIT/p3c1_valid/meta.xml b/Kitodo/src/test/resources/ImportProcessesIT/p3c1_valid/meta.xml new file mode 100644 index 00000000000..4d1b57e258f --- /dev/null +++ b/Kitodo/src/test/resources/ImportProcessesIT/p3c1_valid/meta.xml @@ -0,0 +1,12 @@ + + + + +0001 +Lorem ipsum dolor sit amet +07.08.2023 13:31 + + + +
+ diff --git a/Kitodo/src/test/resources/ImportProcessesIT/p4_valid_butChildIsNot/meta.xml b/Kitodo/src/test/resources/ImportProcessesIT/p4_valid_butChildIsNot/meta.xml new file mode 100644 index 00000000000..3104b0433ae --- /dev/null +++ b/Kitodo/src/test/resources/ImportProcessesIT/p4_valid_butChildIsNot/meta.xml @@ -0,0 +1,14 @@ + + + + +1234 +567 +Lorem ipsum dolor sit amet + + +
+ +
+ +
 diff --git a/Kitodo/src/test/resources/ImportProcessesIT/p4c1_not-valid/meta.xml b/Kitodo/src/test/resources/ImportProcessesIT/p4c1_not-valid/meta.xml new file mode 100644 index 00000000000..ae6653bb878 --- /dev/null +++ b/Kitodo/src/test/resources/ImportProcessesIT/p4c1_not-valid/meta.xml @@ -0,0 +1,11 @@ + + + + +0003 +li cloblwl Shbal + + + +
+ diff --git a/Kitodo/src/test/resources/rulesets/ImportProcessesIT.xml b/Kitodo/src/test/resources/rulesets/ImportProcessesIT.xml new file mode 100644 index 00000000000..cbd10098727 --- /dev/null +++ b/Kitodo/src/test/resources/rulesets/ImportProcessesIT.xml @@ -0,0 +1,100 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + [1-9]\d* + + + + + [1-9]\d{0,4} + + + + \d{4} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +