diff --git a/src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditorAddFilesTest.java b/src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditorAddFilesTest.java index ec36d98..ce7edac 100644 --- a/src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditorAddFilesTest.java +++ b/src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditorAddFilesTest.java @@ -22,13 +22,16 @@ import nl.knaw.dans.dvingest.core.yaml.tasklog.EditFilesLog; import nl.knaw.dans.lib.dataverse.model.dataset.FileList; import nl.knaw.dans.lib.dataverse.model.file.FileMeta; +import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.List; import java.util.UUID; +import java.util.zip.ZipFile; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; @@ -39,11 +42,43 @@ import static org.mockito.Mockito.when; public class FilesEditorAddFilesTest extends FilesEditorTestFixture { + /** + * Creates a FileList based on the entries in a ZIP file, mimicking the behavior of the Dataverse API. + * + * @param zipFile the ZIP file + * @param restricted whether the files are restricted + * @return the FileList + * @throws Exception if an I/O error occurs + */ + private FileList createFileMetaPerZipEntry(Path zipFile, boolean restricted) throws Exception { + var metas = new ArrayList(); + try (var zipFiles = new ZipFile(zipFile.toFile())) { + for (var entry = zipFiles.entries(); entry.hasMoreElements(); ) { + var zipEntry = entry.nextElement(); + var dataversePath = new DataversePath(zipEntry.getName()); + var fileMeta = new FileMeta(); + fileMeta.setLabel(dataversePath.getLabel()); + fileMeta.setDirectoryLabel(dataversePath.getDirectoryLabel()); + fileMeta.setRestricted(restricted); + metas.add(fileMeta); + } + return new FileList(metas); + } + } + + private void assertZipFileContainsFiles(Path zipFile, String... files) throws Exception { + try (var zipFiles = new ZipFile(zipFile.toFile())) { + for (var entry = zipFiles.entries(); entry.hasMoreElements(); ) { + var zipEntry = entry.nextElement(); + assertThat(zipEntry.getName()).isIn(files); + } + } + } @Test public void addRestrictedFiles_adds_one_batch() throws Exception { // Given - Files.createDirectory(testDir.resolve("temp")); + var tempDir = Files.createDirectory(testDir.resolve("temp")); UtilityServices utilityServices = UtilityServicesImpl.builder() .tempDir(testDir.resolve("temp")) .maxUploadSize(1000000) @@ -54,8 +89,12 @@ public void addRestrictedFiles_adds_one_batch() throws Exception { when(dataverseServiceMock.getFiles("pid")).thenReturn( List.of()); - when(dataverseServiceMock.addFile(anyString(), any(Path.class), any(FileMeta.class))).thenReturn( - new FileList(List.of(file("file1", 1), file("file2", 2), file("file3", 3)))); + when(dataverseServiceMock.addFile(anyString(), any(Path.class), any(FileMeta.class))).thenAnswer( + invocation -> { + var path = invocation.getArgument(1, Path.class); + FileUtils.copyFile(path.toFile(), path.resolveSibling("batch.zip").toFile()); + return createFileMetaPerZipEntry(path, true); + }); var editFilesRoot = yamlService.readYamlFromString(""" editFiles: addRestrictedFiles: @@ -73,6 +112,7 @@ public void addRestrictedFiles_adds_one_batch() throws Exception { ArgumentCaptor pathCaptor = ArgumentCaptor.forClass(Path.class); ArgumentCaptor fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class); verify(dataverseServiceMock).addFile(eq("pid"), pathCaptor.capture(), fileMetaCaptor.capture()); + assertThat(fileMetaCaptor.getValue().getRestricted()).isTrue(); assertThat(pathCaptor.getValue().toString()) .withFailMessage("Uploaded file should be a ZIP file in the temp directory") .contains(testDir.resolve("temp").toString()) @@ -80,7 +120,7 @@ public void addRestrictedFiles_adds_one_batch() throws Exception { assertThat(pathCaptor.getValue()) .withFailMessage("ZIP file should be deleted after upload") .doesNotExist(); - assertThat(fileMetaCaptor.getValue().getRestricted()).isTrue(); + assertZipFileContainsFiles(tempDir.resolve("batch.zip"), "file1", "file2", "file3"); YamlBeanAssert.assertThat(editFilesLog.getAddRestrictedFiles()).isEqualTo(""" numberCompleted: 3 completed: true @@ -90,7 +130,7 @@ public void addRestrictedFiles_adds_one_batch() throws Exception { @Test public void addRestrictedFiles_adds_two_batches() throws Exception { // Given - Files.createDirectory(testDir.resolve("temp")); + var tempDir = Files.createDirectory(testDir.resolve("temp")); UtilityServices utilityServices = UtilityServicesImpl.builder() .tempDir(testDir.resolve("temp")) .maxUploadSize(1000000) @@ -101,11 +141,20 @@ public void addRestrictedFiles_adds_two_batches() throws Exception { when(dataverseServiceMock.getFiles("pid")).thenReturn( List.of()); + /* + * (Ab?)using the "thenAnswer" to save then upload the ZIP file, because it is deleted after upload. + */ when(dataverseServiceMock.addFile(anyString(), any(Path.class), any(FileMeta.class))) - // First call - .thenReturn(new FileList(List.of(file("file1", 1), file("file2", 2)))) - // Second call - .thenReturn(new FileList(List.of(file("file3", 3)))); + .thenAnswer(invocation -> { + var path = invocation.getArgument(1, Path.class); + FileUtils.copyFile(path.toFile(), path.resolveSibling("batch1.zip").toFile()); + return createFileMetaPerZipEntry(path, true); + }).thenAnswer( + invocation -> { + var path = invocation.getArgument(1, Path.class); + FileUtils.copyFile(path.toFile(), path.resolveSibling("batch2.zip").toFile()); + return createFileMetaPerZipEntry(path, true); + }); var editFilesRoot = yamlService.readYamlFromString(""" editFiles: addRestrictedFiles: @@ -123,6 +172,7 @@ public void addRestrictedFiles_adds_two_batches() throws Exception { ArgumentCaptor pathCaptor = ArgumentCaptor.forClass(Path.class); ArgumentCaptor fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class); verify(dataverseServiceMock, times(2)).addFile(eq("pid"), pathCaptor.capture(), fileMetaCaptor.capture()); + assertThat(fileMetaCaptor.getValue().getRestricted()).isTrue(); assertThat(pathCaptor.getValue().toString()) .withFailMessage("Uploaded file should be a ZIP file in the temp directory") .contains(testDir.resolve("temp").toString()) @@ -130,11 +180,181 @@ public void addRestrictedFiles_adds_two_batches() throws Exception { assertThat(pathCaptor.getValue()) .withFailMessage("ZIP file should be deleted after upload") .doesNotExist(); - assertThat(fileMetaCaptor.getValue().getRestricted()).isTrue(); + assertZipFileContainsFiles(tempDir.resolve("batch1.zip"), "file1", "file2"); + assertZipFileContainsFiles(tempDir.resolve("batch2.zip"), "file3"); YamlBeanAssert.assertThat(editFilesLog.getAddRestrictedFiles()).isEqualTo(""" numberCompleted: 3 completed: true """); } + @Test + public void addUnrestrictedFiles_adds_one_batch() throws Exception { + // Given + var tempDir = Files.createDirectory(testDir.resolve("temp")); + UtilityServices utilityServices = UtilityServicesImpl.builder() + .tempDir(testDir.resolve("temp")) + .maxUploadSize(1000000) + .maxNumberOfFilesPerUpload(100).build(); + Files.createFile(dataDir.resolve("file1")); + Files.createFile(dataDir.resolve("file2")); + Files.createFile(dataDir.resolve("file3")); + + when(dataverseServiceMock.getFiles("pid")).thenReturn( + List.of()); + when(dataverseServiceMock.addFile(anyString(), any(Path.class), any(FileMeta.class))).thenAnswer( + invocation -> { + var path = invocation.getArgument(1, Path.class); + FileUtils.copyFile(path.toFile(), path.resolveSibling("batch.zip").toFile()); + return createFileMetaPerZipEntry(path, false); + }); + var editFilesRoot = yamlService.readYamlFromString(""" + editFiles: + addUnrestrictedFiles: + - file1 + - file2 + - file3 + """, EditFilesRoot.class); + var editFilesLog = new EditFilesLog(); + var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServices, editFilesLog); + + // When + filesEditor.editFiles("pid"); + + // Then + ArgumentCaptor pathCaptor = ArgumentCaptor.forClass(Path.class); + ArgumentCaptor fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class); + verify(dataverseServiceMock).addFile(eq("pid"), pathCaptor.capture(), fileMetaCaptor.capture()); + assertThat(fileMetaCaptor.getValue().getRestricted()).isFalse(); + assertThat(pathCaptor.getValue().toString()) + .withFailMessage("Uploaded file should be a ZIP file in the temp directory") + .contains(testDir.resolve("temp").toString()) + .endsWith(".zip"); + assertThat(pathCaptor.getValue()) + .withFailMessage("ZIP file should be deleted after upload") + .doesNotExist(); + assertZipFileContainsFiles(tempDir.resolve("batch.zip"), "file1", "file2", "file3"); + YamlBeanAssert.assertThat(editFilesLog.getAddUnrestrictedFiles()).isEqualTo(""" + numberCompleted: 3 + completed: true + """); + } + + @Test + public void addUnrestrictedFiles_adds_two_batches() throws Exception { + // Given + var tempDir = Files.createDirectory(testDir.resolve("temp")); + UtilityServices utilityServices = UtilityServicesImpl.builder() + .tempDir(testDir.resolve("temp")) + .maxUploadSize(1000000) + .maxNumberOfFilesPerUpload(2).build(); // Causes first batch to be limited to 2 files + Files.createFile(dataDir.resolve("file1")); + Files.createFile(dataDir.resolve("file2")); + Files.createFile(dataDir.resolve("file3")); + + when(dataverseServiceMock.getFiles("pid")).thenReturn( + List.of()); + /* + * (Ab?)using the "thenAnswer" to save then upload the ZIP file, because it is deleted after upload. + */ + when(dataverseServiceMock.addFile(anyString(), any(Path.class), any(FileMeta.class))) + .thenAnswer(invocation -> { + var path = invocation.getArgument(1, Path.class); + FileUtils.copyFile(path.toFile(), path.resolveSibling("batch1.zip").toFile()); + return createFileMetaPerZipEntry(path, false); + }).thenAnswer( + invocation -> { + var path = invocation.getArgument(1, Path.class); + FileUtils.copyFile(path.toFile(), path.resolveSibling("batch2.zip").toFile()); + return createFileMetaPerZipEntry(path, false); + }); + var editFilesRoot = yamlService.readYamlFromString(""" + editFiles: + addUnrestrictedFiles: + - file1 + - file2 + - file3 + """, EditFilesRoot.class); + var editFilesLog = new EditFilesLog(); + var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServices, editFilesLog); + + // When + filesEditor.editFiles("pid"); + + // Then + ArgumentCaptor pathCaptor = ArgumentCaptor.forClass(Path.class); + ArgumentCaptor fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class); + verify(dataverseServiceMock, times(2)).addFile(eq("pid"), pathCaptor.capture(), fileMetaCaptor.capture()); + assertThat(fileMetaCaptor.getValue().getRestricted()).isFalse(); + assertThat(pathCaptor.getValue().toString()) + .withFailMessage("Uploaded file should be a ZIP file in the temp directory") + .contains(testDir.resolve("temp").toString()) + .endsWith(".zip"); + assertThat(pathCaptor.getValue()) + .withFailMessage("ZIP file should be deleted after upload") + .doesNotExist(); + assertZipFileContainsFiles(tempDir.resolve("batch1.zip"), "file1", "file2"); + assertZipFileContainsFiles(tempDir.resolve("batch2.zip"), "file3"); + YamlBeanAssert.assertThat(editFilesLog.getAddUnrestrictedFiles()).isEqualTo(""" + numberCompleted: 3 + completed: true + """); + } + + @Test + public void addUnrestrictedFiles_skips_already_completed_files() throws Exception { + // Given + var tempDir = Files.createDirectory(testDir.resolve("temp")); + UtilityServices utilityServices = UtilityServicesImpl.builder() + .tempDir(testDir.resolve("temp")) + .maxUploadSize(1000000) + .maxNumberOfFilesPerUpload(2).build(); // Causes first batch to be limited to 2 files + Files.createFile(dataDir.resolve("file1")); + Files.createFile(dataDir.resolve("file2")); + Files.createFile(dataDir.resolve("file3")); + + when(dataverseServiceMock.getFiles("pid")).thenReturn( + List.of()); + /* + * (Ab?)using the "thenAnswer" to save then upload the ZIP file, because it is deleted after upload. + */ + when(dataverseServiceMock.addFile(anyString(), any(Path.class), any(FileMeta.class))) + .thenAnswer(invocation -> { + var path = invocation.getArgument(1, Path.class); + FileUtils.copyFile(path.toFile(), path.resolveSibling("batch1.zip").toFile()); + return createFileMetaPerZipEntry(path, false); + }); + var editFilesRoot = yamlService.readYamlFromString(""" + editFiles: + addUnrestrictedFiles: + - file1 + - file2 + - file3 + """, EditFilesRoot.class); + var editFilesLog = new EditFilesLog(); + editFilesLog.getAddUnrestrictedFiles().setNumberCompleted(1); + var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServices, editFilesLog); + + // When + filesEditor.editFiles("pid"); + + // Then + ArgumentCaptor pathCaptor = ArgumentCaptor.forClass(Path.class); + ArgumentCaptor fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class); + verify(dataverseServiceMock).addFile(eq("pid"), pathCaptor.capture(), fileMetaCaptor.capture()); + assertThat(fileMetaCaptor.getValue().getRestricted()).isFalse(); + assertThat(pathCaptor.getValue().toString()) + .withFailMessage("Uploaded file should be a ZIP file in the temp directory") + .contains(testDir.resolve("temp").toString()) + .endsWith(".zip"); + assertThat(pathCaptor.getValue()) + .withFailMessage("ZIP file should be deleted after upload") + .doesNotExist(); + assertZipFileContainsFiles(tempDir.resolve("batch1.zip"), "file2", "file3"); + YamlBeanAssert.assertThat(editFilesLog.getAddUnrestrictedFiles()).isEqualTo(""" + numberCompleted: 3 + completed: true + """); + } + }