Skip to content

Commit

Permalink
more testing
Browse files Browse the repository at this point in the history
  • Loading branch information
@janvanmansum
janvanmansum committed Jan 20, 2025
1 parent cef87e5 commit e4984e4
Showing 1 changed file with 230 additions and 10 deletions.
240 changes: 230 additions & 10 deletions src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditorAddFilesTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,16 @@
import nl.knaw.dans.dvingest.core.yaml.tasklog.EditFilesLog;
import nl.knaw.dans.lib.dataverse.model.dataset.FileList;
import nl.knaw.dans.lib.dataverse.model.file.FileMeta;
import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;

import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.zip.ZipFile;

import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
Expand All @@ -39,11 +42,43 @@
import static org.mockito.Mockito.when;

public class FilesEditorAddFilesTest extends FilesEditorTestFixture {
/**
* Creates a FileList based on the entries in a ZIP file, mimicking the behavior of the Dataverse API.
*
* @param zipFile the ZIP file
* @param restricted whether the files are restricted
* @return the FileList
* @throws Exception if an I/O error occurs
*/
private FileList createFileMetaPerZipEntry(Path zipFile, boolean restricted) throws Exception {
var metas = new ArrayList<FileMeta>();
try (var zipFiles = new ZipFile(zipFile.toFile())) {
for (var entry = zipFiles.entries(); entry.hasMoreElements(); ) {
var zipEntry = entry.nextElement();
var dataversePath = new DataversePath(zipEntry.getName());
var fileMeta = new FileMeta();
fileMeta.setLabel(dataversePath.getLabel());
fileMeta.setDirectoryLabel(dataversePath.getDirectoryLabel());
fileMeta.setRestricted(restricted);
metas.add(fileMeta);
}
return new FileList(metas);
}
}

private void assertZipFileContainsFiles(Path zipFile, String... files) throws Exception {
try (var zipFiles = new ZipFile(zipFile.toFile())) {
for (var entry = zipFiles.entries(); entry.hasMoreElements(); ) {
var zipEntry = entry.nextElement();
assertThat(zipEntry.getName()).isIn(files);
}
}
}

@Test
public void addRestrictedFiles_adds_one_batch() throws Exception {
// Given
Files.createDirectory(testDir.resolve("temp"));
var tempDir = Files.createDirectory(testDir.resolve("temp"));
UtilityServices utilityServices = UtilityServicesImpl.builder()
.tempDir(testDir.resolve("temp"))
.maxUploadSize(1000000)
Expand All @@ -54,8 +89,12 @@ public void addRestrictedFiles_adds_one_batch() throws Exception {

when(dataverseServiceMock.getFiles("pid")).thenReturn(
List.of());
when(dataverseServiceMock.addFile(anyString(), any(Path.class), any(FileMeta.class))).thenReturn(
new FileList(List.of(file("file1", 1), file("file2", 2), file("file3", 3))));
when(dataverseServiceMock.addFile(anyString(), any(Path.class), any(FileMeta.class))).thenAnswer(
invocation -> {
var path = invocation.getArgument(1, Path.class);
FileUtils.copyFile(path.toFile(), path.resolveSibling("batch.zip").toFile());
return createFileMetaPerZipEntry(path, true);
});
var editFilesRoot = yamlService.readYamlFromString("""
editFiles:
addRestrictedFiles:
Expand All @@ -73,14 +112,15 @@ public void addRestrictedFiles_adds_one_batch() throws Exception {
ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
ArgumentCaptor<FileMeta> fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class);
verify(dataverseServiceMock).addFile(eq("pid"), pathCaptor.capture(), fileMetaCaptor.capture());
assertThat(fileMetaCaptor.getValue().getRestricted()).isTrue();
assertThat(pathCaptor.getValue().toString())
.withFailMessage("Uploaded file should be a ZIP file in the temp directory")
.contains(testDir.resolve("temp").toString())
.endsWith(".zip");
assertThat(pathCaptor.getValue())
.withFailMessage("ZIP file should be deleted after upload")
.doesNotExist();
assertThat(fileMetaCaptor.getValue().getRestricted()).isTrue();
assertZipFileContainsFiles(tempDir.resolve("batch.zip"), "file1", "file2", "file3");
YamlBeanAssert.assertThat(editFilesLog.getAddRestrictedFiles()).isEqualTo("""
numberCompleted: 3
completed: true
Expand All @@ -90,7 +130,7 @@ public void addRestrictedFiles_adds_one_batch() throws Exception {
@Test
public void addRestrictedFiles_adds_two_batches() throws Exception {
// Given
Files.createDirectory(testDir.resolve("temp"));
var tempDir = Files.createDirectory(testDir.resolve("temp"));
UtilityServices utilityServices = UtilityServicesImpl.builder()
.tempDir(testDir.resolve("temp"))
.maxUploadSize(1000000)
Expand All @@ -101,11 +141,20 @@ public void addRestrictedFiles_adds_two_batches() throws Exception {

when(dataverseServiceMock.getFiles("pid")).thenReturn(
List.of());
/*
* (Ab?)using the "thenAnswer" to save then upload the ZIP file, because it is deleted after upload.
*/
when(dataverseServiceMock.addFile(anyString(), any(Path.class), any(FileMeta.class)))
// First call
.thenReturn(new FileList(List.of(file("file1", 1), file("file2", 2))))
// Second call
.thenReturn(new FileList(List.of(file("file3", 3))));
.thenAnswer(invocation -> {
var path = invocation.getArgument(1, Path.class);
FileUtils.copyFile(path.toFile(), path.resolveSibling("batch1.zip").toFile());
return createFileMetaPerZipEntry(path, true);
}).thenAnswer(
invocation -> {
var path = invocation.getArgument(1, Path.class);
FileUtils.copyFile(path.toFile(), path.resolveSibling("batch2.zip").toFile());
return createFileMetaPerZipEntry(path, true);
});
var editFilesRoot = yamlService.readYamlFromString("""
editFiles:
addRestrictedFiles:
Expand All @@ -123,18 +172,189 @@ public void addRestrictedFiles_adds_two_batches() throws Exception {
ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
ArgumentCaptor<FileMeta> fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class);
verify(dataverseServiceMock, times(2)).addFile(eq("pid"), pathCaptor.capture(), fileMetaCaptor.capture());
assertThat(fileMetaCaptor.getValue().getRestricted()).isTrue();
assertThat(pathCaptor.getValue().toString())
.withFailMessage("Uploaded file should be a ZIP file in the temp directory")
.contains(testDir.resolve("temp").toString())
.endsWith(".zip");
assertThat(pathCaptor.getValue())
.withFailMessage("ZIP file should be deleted after upload")
.doesNotExist();
assertThat(fileMetaCaptor.getValue().getRestricted()).isTrue();
assertZipFileContainsFiles(tempDir.resolve("batch1.zip"), "file1", "file2");
assertZipFileContainsFiles(tempDir.resolve("batch2.zip"), "file3");
YamlBeanAssert.assertThat(editFilesLog.getAddRestrictedFiles()).isEqualTo("""
numberCompleted: 3
completed: true
""");
}

@Test
public void addUnrestrictedFiles_adds_one_batch() throws Exception {
// Given
var tempDir = Files.createDirectory(testDir.resolve("temp"));
UtilityServices utilityServices = UtilityServicesImpl.builder()
.tempDir(testDir.resolve("temp"))
.maxUploadSize(1000000)
.maxNumberOfFilesPerUpload(100).build();
Files.createFile(dataDir.resolve("file1"));
Files.createFile(dataDir.resolve("file2"));
Files.createFile(dataDir.resolve("file3"));

when(dataverseServiceMock.getFiles("pid")).thenReturn(
List.of());
when(dataverseServiceMock.addFile(anyString(), any(Path.class), any(FileMeta.class))).thenAnswer(
invocation -> {
var path = invocation.getArgument(1, Path.class);
FileUtils.copyFile(path.toFile(), path.resolveSibling("batch.zip").toFile());
return createFileMetaPerZipEntry(path, false);
});
var editFilesRoot = yamlService.readYamlFromString("""
editFiles:
addUnrestrictedFiles:
- file1
- file2
- file3
""", EditFilesRoot.class);
var editFilesLog = new EditFilesLog();
var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServices, editFilesLog);

// When
filesEditor.editFiles("pid");

// Then
ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
ArgumentCaptor<FileMeta> fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class);
verify(dataverseServiceMock).addFile(eq("pid"), pathCaptor.capture(), fileMetaCaptor.capture());
assertThat(fileMetaCaptor.getValue().getRestricted()).isFalse();
assertThat(pathCaptor.getValue().toString())
.withFailMessage("Uploaded file should be a ZIP file in the temp directory")
.contains(testDir.resolve("temp").toString())
.endsWith(".zip");
assertThat(pathCaptor.getValue())
.withFailMessage("ZIP file should be deleted after upload")
.doesNotExist();
assertZipFileContainsFiles(tempDir.resolve("batch.zip"), "file1", "file2", "file3");
YamlBeanAssert.assertThat(editFilesLog.getAddUnrestrictedFiles()).isEqualTo("""
numberCompleted: 3
completed: true
""");
}

@Test
public void addUnrestrictedFiles_adds_two_batches() throws Exception {
// Given
var tempDir = Files.createDirectory(testDir.resolve("temp"));
UtilityServices utilityServices = UtilityServicesImpl.builder()
.tempDir(testDir.resolve("temp"))
.maxUploadSize(1000000)
.maxNumberOfFilesPerUpload(2).build(); // Causes first batch to be limited to 2 files
Files.createFile(dataDir.resolve("file1"));
Files.createFile(dataDir.resolve("file2"));
Files.createFile(dataDir.resolve("file3"));

when(dataverseServiceMock.getFiles("pid")).thenReturn(
List.of());
/*
* (Ab?)using the "thenAnswer" to save then upload the ZIP file, because it is deleted after upload.
*/
when(dataverseServiceMock.addFile(anyString(), any(Path.class), any(FileMeta.class)))
.thenAnswer(invocation -> {
var path = invocation.getArgument(1, Path.class);
FileUtils.copyFile(path.toFile(), path.resolveSibling("batch1.zip").toFile());
return createFileMetaPerZipEntry(path, false);
}).thenAnswer(
invocation -> {
var path = invocation.getArgument(1, Path.class);
FileUtils.copyFile(path.toFile(), path.resolveSibling("batch2.zip").toFile());
return createFileMetaPerZipEntry(path, false);
});
var editFilesRoot = yamlService.readYamlFromString("""
editFiles:
addUnrestrictedFiles:
- file1
- file2
- file3
""", EditFilesRoot.class);
var editFilesLog = new EditFilesLog();
var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServices, editFilesLog);

// When
filesEditor.editFiles("pid");

// Then
ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
ArgumentCaptor<FileMeta> fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class);
verify(dataverseServiceMock, times(2)).addFile(eq("pid"), pathCaptor.capture(), fileMetaCaptor.capture());
assertThat(fileMetaCaptor.getValue().getRestricted()).isFalse();
assertThat(pathCaptor.getValue().toString())
.withFailMessage("Uploaded file should be a ZIP file in the temp directory")
.contains(testDir.resolve("temp").toString())
.endsWith(".zip");
assertThat(pathCaptor.getValue())
.withFailMessage("ZIP file should be deleted after upload")
.doesNotExist();
assertZipFileContainsFiles(tempDir.resolve("batch1.zip"), "file1", "file2");
assertZipFileContainsFiles(tempDir.resolve("batch2.zip"), "file3");
YamlBeanAssert.assertThat(editFilesLog.getAddUnrestrictedFiles()).isEqualTo("""
numberCompleted: 3
completed: true
""");
}

@Test
public void addUnrestrictedFiles_skips_already_completed_files() throws Exception {
// Given
var tempDir = Files.createDirectory(testDir.resolve("temp"));
UtilityServices utilityServices = UtilityServicesImpl.builder()
.tempDir(testDir.resolve("temp"))
.maxUploadSize(1000000)
.maxNumberOfFilesPerUpload(2).build(); // Causes first batch to be limited to 2 files
Files.createFile(dataDir.resolve("file1"));
Files.createFile(dataDir.resolve("file2"));
Files.createFile(dataDir.resolve("file3"));

when(dataverseServiceMock.getFiles("pid")).thenReturn(
List.of());
/*
* (Ab?)using the "thenAnswer" to save then upload the ZIP file, because it is deleted after upload.
*/
when(dataverseServiceMock.addFile(anyString(), any(Path.class), any(FileMeta.class)))
.thenAnswer(invocation -> {
var path = invocation.getArgument(1, Path.class);
FileUtils.copyFile(path.toFile(), path.resolveSibling("batch1.zip").toFile());
return createFileMetaPerZipEntry(path, false);
});
var editFilesRoot = yamlService.readYamlFromString("""
editFiles:
addUnrestrictedFiles:
- file1
- file2
- file3
""", EditFilesRoot.class);
var editFilesLog = new EditFilesLog();
editFilesLog.getAddUnrestrictedFiles().setNumberCompleted(1);
var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServices, editFilesLog);

// When
filesEditor.editFiles("pid");

// Then
ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
ArgumentCaptor<FileMeta> fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class);
verify(dataverseServiceMock).addFile(eq("pid"), pathCaptor.capture(), fileMetaCaptor.capture());
assertThat(fileMetaCaptor.getValue().getRestricted()).isFalse();
assertThat(pathCaptor.getValue().toString())
.withFailMessage("Uploaded file should be a ZIP file in the temp directory")
.contains(testDir.resolve("temp").toString())
.endsWith(".zip");
assertThat(pathCaptor.getValue())
.withFailMessage("ZIP file should be deleted after upload")
.doesNotExist();
assertZipFileContainsFiles(tempDir.resolve("batch1.zip"), "file2", "file3");
YamlBeanAssert.assertThat(editFilesLog.getAddUnrestrictedFiles()).isEqualTo("""
numberCompleted: 3
completed: true
""");
}

}

0 comments on commit e4984e4

Please sign in to comment.