From 43b73ab357df8496e10fec660c82eb7cc9beefea Mon Sep 17 00:00:00 2001 From: Jan van Mansum Date: Sun, 19 Jan 2025 12:39:10 +0100 Subject: [PATCH] Tests for addIndividually --- .../core/bagprocessor/FilesEditor.java | 46 ++- .../FilesEditorAddFilesIndividuallyTest.java | 300 ++++++++++++++++++ 2 files changed, 337 insertions(+), 9 deletions(-) create mode 100644 src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditorAddFilesIndividuallyTest.java diff --git a/src/main/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditor.java b/src/main/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditor.java index 1a649fa..426b77a 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditor.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditor.java @@ -192,24 +192,43 @@ private void replaceFileOrThrow(String pid, FileMeta fileMeta, Path fileToUpload } public void addRestrictedFilesIndividually() throws IOException, DataverseException { - addFilesIndividually(editFiles.getAddRestrictedIndividually(), true); + if (editFilesLog.getAddRestrictedIndividually().isCompleted()) { + log.debug("Task addRestrictedIndividually already completed for deposit {}", depositId); + return; + } + if (editFiles.getAddRestrictedIndividually().isEmpty()) { + log.debug("No restricted files to add individually for deposit {}", depositId); + } else { + addFilesIndividually(editFiles.getAddRestrictedIndividually(), true); + } + editFilesLog.getAddRestrictedIndividually().setCompleted(true); } public void addUnrestrictedFilesIndividually() throws IOException, DataverseException { - addFilesIndividually(editFiles.getAddUnrestrictedIndividually(), false); + if (editFilesLog.getAddUnrestrictedIndividually().isCompleted()) { + log.debug("Task addUnrestrictedIndividually already completed for deposit {}", depositId); + return; + } + if (editFiles.getAddUnrestrictedIndividually().isEmpty()) { + log.debug("No unrestricted files to add individually for deposit {}", depositId); + } else { + addFilesIndividually(editFiles.getAddUnrestrictedIndividually(), false); + } + editFilesLog.getAddUnrestrictedIndividually().setCompleted(true); } public void addFilesIndividually(List files, boolean restricted) throws IOException, DataverseException { - if (files.isEmpty()) { - log.debug("No files to add individually for deposit {}, restrict = {}", depositId, restricted); - return; - } log.debug("Start adding {} files individually for deposit {}, restrict = {}", editFiles.getAddRestrictedIndividually().size(), depositId, restricted); - for (var filepath : files) { - log.debug("Adding restricted file: {}", filepath); + int numberAdded = editFilesLog.getAddRestrictedIndividually().getNumberCompleted(); + if (numberAdded > 0) { + log.debug("Resuming adding files from number {}", numberAdded); + } + for (int i = numberAdded; i < files.size(); i++) { + var filepath = files.get(i); + log.debug("Adding file: {}", filepath); var fileMeta = new FileMeta(); fileMeta.setRestricted(restricted); - String realFilepath = filepath; + var realFilepath = filepath; // TODO: a bit confusing that autorenamedFiles is part of the cache, although the file looked up here has not been added to the dataset yet. if (filesInDatasetCache.getAutoRenamedFiles().containsKey(filepath)) { realFilepath = filesInDatasetCache.getAutoRenamedFiles().get(filepath); @@ -218,10 +237,19 @@ public void addFilesIndividually(List files, boolean restricted) throws fileMeta.setLabel(dataversePath.getLabel()); fileMeta.setDirectoryLabel(dataversePath.getDirectoryLabel()); var fileToUpload = dataDir.resolve(filepath); + if (!Files.exists(fileToUpload)) { + throw new IllegalArgumentException("File to add not found in bag: " + filepath); + } var addedFileMeta = dataverseService.addFile(pid, fileToUpload, fileMeta); for (var fm : addedFileMeta.getFiles()) { filesInDatasetCache.put(fm); } + if (restricted) { + editFilesLog.getAddRestrictedIndividually().setNumberCompleted(++numberAdded); + } else { + editFilesLog.getAddUnrestrictedIndividually().setNumberCompleted(++numberAdded); + } + } } diff --git a/src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditorAddFilesIndividuallyTest.java b/src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditorAddFilesIndividuallyTest.java new file mode 100644 index 0000000..9bd188b --- /dev/null +++ b/src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditorAddFilesIndividuallyTest.java @@ -0,0 +1,300 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvingest.core.bagprocessor; + +import nl.knaw.dans.dvingest.YamlBeanAssert; +import nl.knaw.dans.dvingest.core.yaml.EditFilesRoot; +import nl.knaw.dans.dvingest.core.yaml.tasklog.EditFilesLog; +import nl.knaw.dans.lib.dataverse.model.dataset.FileList; +import nl.knaw.dans.lib.dataverse.model.file.FileMeta; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +public class FilesEditorAddFilesIndividuallyTest extends FilesEditorTestFixture { + @Test + public void addRestrictedIndividually_adds_files_to_dataset() throws Exception { + // Given + ArgumentCaptor fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class); + when(dataverseServiceMock.addFile(anyString(), any(Path.class), fileMetaCaptor.capture())) + .thenAnswer(invocation -> new FileList(List.of(fileMetaCaptor.getValue()))); + Files.createFile(dataDir.resolve("file1")); + Files.createFile(dataDir.resolve("file2")); + var editFilesRoot = yamlService.readYamlFromString(""" + editFiles: + addRestrictedIndividually: + - file1 + - file2 + """, EditFilesRoot.class); + var editFilesLog = new EditFilesLog(); + var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServicesMock, editFilesLog); + + // When + filesEditor.editFiles("pid"); + + // Then + Mockito.verify(dataverseServiceMock).addFile(anyString(), eq(dataDir.resolve("file1")), Mockito.any(FileMeta.class)); + Mockito.verify(dataverseServiceMock).addFile(anyString(), eq(dataDir.resolve("file2")), Mockito.any(FileMeta.class)); + YamlBeanAssert.assertThat(editFilesLog.getAddRestrictedIndividually()).isEqualTo(""" + numberCompleted: 2 + completed: true + """); + } + + @Test + public void addUnrestrictedFilesIndividually_adds_files_to_dataset() throws Exception { + // Given + ArgumentCaptor fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class); + when(dataverseServiceMock.addFile(anyString(), any(Path.class), fileMetaCaptor.capture())) + .thenAnswer(invocation -> new FileList(List.of(fileMetaCaptor.getValue()))); + Files.createFile(dataDir.resolve("file3")); + Files.createFile(dataDir.resolve("file4")); + var editFilesRoot = yamlService.readYamlFromString(""" + editFiles: + addUnrestrictedIndividually: + - file3 + - file4 + """, EditFilesRoot.class); + var editFilesLog = new EditFilesLog(); + var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServicesMock, editFilesLog); + + // When + filesEditor.editFiles("pid"); + + // Then + Mockito.verify(dataverseServiceMock).addFile(anyString(), eq(dataDir.resolve("file3")), Mockito.any(FileMeta.class)); + Mockito.verify(dataverseServiceMock).addFile(anyString(), eq(dataDir.resolve("file4")), Mockito.any(FileMeta.class)); + YamlBeanAssert.assertThat(editFilesLog.getAddUnrestrictedIndividually()).isEqualTo(""" + numberCompleted: 2 + completed: true + """); + } + + @Test + public void addRestrictedFilesIndividually_throws_exception_when_file_not_found() throws Exception { + // Given + ArgumentCaptor fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class); + when(dataverseServiceMock.addFile(anyString(), any(Path.class), fileMetaCaptor.capture())) + .thenAnswer(invocation -> new FileList(List.of(fileMetaCaptor.getValue()))); + var editFilesRoot = yamlService.readYamlFromString(""" + editFiles: + addRestrictedIndividually: + - file1 + - file5 + """, EditFilesRoot.class); + // Create file1 but not file5 + Files.createFile(dataDir.resolve("file1")); + var editFilesLog = new EditFilesLog(); + var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServicesMock, editFilesLog); + + // When + assertThatThrownBy(() -> filesEditor.editFiles("pid")) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("File to add not found in bag: file5"); + YamlBeanAssert.assertThat(editFilesLog.getAddRestrictedIndividually()).isEqualTo(""" + numberCompleted: 1 + completed: false + """); + } + + @Test + public void addUnrestrictedFilesIndividually_throws_exception_when_file_not_found() throws Exception { + // Given + ArgumentCaptor fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class); + when(dataverseServiceMock.addFile(anyString(), any(Path.class), fileMetaCaptor.capture())) + .thenAnswer(invocation -> new FileList(List.of(fileMetaCaptor.getValue()))); + var editFilesRoot = yamlService.readYamlFromString(""" + editFiles: + addUnrestrictedIndividually: + - file3 + - file6 + """, EditFilesRoot.class); + // create file3 but not file6 + Files.createFile(dataDir.resolve("file3")); + var editFilesLog = new EditFilesLog(); + var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServicesMock, editFilesLog); + + // When + assertThatThrownBy(() -> filesEditor.editFiles("pid")) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("File to add not found in bag: file6"); + YamlBeanAssert.assertThat(editFilesLog.getAddUnrestrictedIndividually()).isEqualTo(""" + numberCompleted: 1 + completed: false + """); + } + + @Test + public void addRestrictedFilesIndividually_is_skipped_when_already_completed() throws Exception { + // Given + var editFilesRoot = yamlService.readYamlFromString(""" + editFiles: + addRestrictedIndividually: + - file1 + - file2 + """, EditFilesRoot.class); + var editFilesLog = new EditFilesLog(); + editFilesLog.getAddRestrictedIndividually().setCompleted(true); + editFilesLog.getAddRestrictedIndividually().setNumberCompleted(2); + var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServicesMock, editFilesLog); + + // When + filesEditor.addRestrictedFilesIndividually(); + + // Then + Mockito.verify(dataverseServiceMock, Mockito.never()).addFile(Mockito.anyString(), Mockito.any(Path.class), Mockito.any(FileMeta.class)); + YamlBeanAssert.assertThat(editFilesLog.getAddRestrictedIndividually()).isEqualTo(""" + numberCompleted: 2 + completed: true + """); + } + + @Test + public void addUnrestrictedFilesIndividually_is_skipped_when_already_completed() throws Exception { + // Given + var editFilesRoot = yamlService.readYamlFromString(""" + editFiles: + addUnrestrictedIndividually: + - file3 + - file4 + """, EditFilesRoot.class); + var editFilesLog = new EditFilesLog(); + editFilesLog.getAddUnrestrictedIndividually().setCompleted(true); + editFilesLog.getAddUnrestrictedIndividually().setNumberCompleted(2); + var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServicesMock, editFilesLog); + + // When + filesEditor.addUnrestrictedFilesIndividually(); + + // Then + Mockito.verify(dataverseServiceMock, Mockito.never()).addFile(Mockito.anyString(), Mockito.any(Path.class), Mockito.any(FileMeta.class)); + YamlBeanAssert.assertThat(editFilesLog.getAddUnrestrictedIndividually()).isEqualTo(""" + numberCompleted: 2 + completed: true + """); + } + + @Test + public void addRestrictedFilesIndividually_will_continue_after_number_already_completed() throws Exception { + // Given + ArgumentCaptor fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class); + when(dataverseServiceMock.addFile(anyString(), any(Path.class), fileMetaCaptor.capture())) + .thenAnswer(invocation -> new FileList(List.of(fileMetaCaptor.getValue()))); + Files.createFile(dataDir.resolve("file1")); + Files.createFile(dataDir.resolve("file2")); + Files.createFile(dataDir.resolve("file3")); + var editFilesRoot = yamlService.readYamlFromString(""" + editFiles: + addRestrictedIndividually: + - file1 + - file2 + - file3 + """, EditFilesRoot.class); + var editFilesLog = new EditFilesLog(); + editFilesLog.getAddRestrictedIndividually().setCompleted(false); + editFilesLog.getAddRestrictedIndividually().setNumberCompleted(2); + var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServicesMock, editFilesLog); + + // When + filesEditor.editFiles("pid"); + + // Then + Mockito.verify(dataverseServiceMock).addFile(anyString(), eq(dataDir.resolve("file3")), Mockito.any(FileMeta.class)); + YamlBeanAssert.assertThat(editFilesLog.getAddRestrictedIndividually()).isEqualTo(""" + numberCompleted: 3 + completed: true + """); + } + + // The same for unrestricted + @Test + public void addUnrestrictedFilesIndividually_will_continue_after_number_already_completed() throws Exception { + // Given + ArgumentCaptor fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class); + when(dataverseServiceMock.addFile(anyString(), any(Path.class), fileMetaCaptor.capture())) + .thenAnswer(invocation -> new FileList(List.of(fileMetaCaptor.getValue()))); + Files.createFile(dataDir.resolve("file1")); + Files.createFile(dataDir.resolve("file2")); + Files.createFile(dataDir.resolve("file3")); + var editFilesRoot = yamlService.readYamlFromString(""" + editFiles: + addUnrestrictedIndividually: + - file1 + - file2 + - file3 + """, EditFilesRoot.class); + var editFilesLog = new EditFilesLog(); + editFilesLog.getAddUnrestrictedIndividually().setCompleted(false); + editFilesLog.getAddUnrestrictedIndividually().setNumberCompleted(2); + var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServicesMock, editFilesLog); + + // When + filesEditor.editFiles("pid"); + + // Then + Mockito.verify(dataverseServiceMock).addFile(anyString(), eq(dataDir.resolve("file3")), Mockito.any(FileMeta.class)); + YamlBeanAssert.assertThat(editFilesLog.getAddUnrestrictedIndividually()).isEqualTo(""" + numberCompleted: 3 + completed: true + """); + } + + @Test + public void addRestrictedFilesIndividually_will_auto_rename_file() throws Exception { + // Given + ArgumentCaptor fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class); + when(dataverseServiceMock.addFile(anyString(), any(Path.class), fileMetaCaptor.capture())) + .thenAnswer(invocation -> new FileList(List.of(fileMetaCaptor.getValue()))); + Files.createFile(dataDir.resolve("file1")); + Files.createFile(dataDir.resolve("file2")); + Files.createFile(dataDir.resolve("file3")); + var editFilesRoot = yamlService.readYamlFromString(""" + editFiles: + addRestrictedIndividually: + - file1 + - file2 + - file3 + autoRenameFiles: + - from: file3 + to: file3-renamed + """, EditFilesRoot.class); + var editFilesLog = new EditFilesLog(); + var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServicesMock, editFilesLog); + // When + filesEditor.editFiles("pid"); + + // Then + Mockito.verify(dataverseServiceMock).addFile(eq("pid"), eq(dataDir.resolve("file3")), fileMetaCaptor.capture()); + assertThat(fileMetaCaptor.getValue().getLabel()).isEqualTo("file3-renamed"); + YamlBeanAssert.assertThat(editFilesLog.getAddRestrictedIndividually()).isEqualTo(""" + numberCompleted: 3 + completed: true + """); + } + +}