Skip to content

Commit

Permalink
Tests for addIndividually
Browse files Browse the repository at this point in the history
  • Loading branch information
@janvanmansum
janvanmansum committed Jan 19, 2025
1 parent 72d1d22 commit 43b73ab
Show file tree
Hide file tree
Showing 2 changed files with 337 additions and 9 deletions.
46 changes: 37 additions & 9 deletions src/main/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,24 +192,43 @@ private void replaceFileOrThrow(String pid, FileMeta fileMeta, Path fileToUpload
}

public void addRestrictedFilesIndividually() throws IOException, DataverseException {
addFilesIndividually(editFiles.getAddRestrictedIndividually(), true);
if (editFilesLog.getAddRestrictedIndividually().isCompleted()) {
log.debug("Task addRestrictedIndividually already completed for deposit {}", depositId);
return;
}
if (editFiles.getAddRestrictedIndividually().isEmpty()) {
log.debug("No restricted files to add individually for deposit {}", depositId);
} else {
addFilesIndividually(editFiles.getAddRestrictedIndividually(), true);
}
editFilesLog.getAddRestrictedIndividually().setCompleted(true);
}

public void addUnrestrictedFilesIndividually() throws IOException, DataverseException {
addFilesIndividually(editFiles.getAddUnrestrictedIndividually(), false);
if (editFilesLog.getAddUnrestrictedIndividually().isCompleted()) {
log.debug("Task addUnrestrictedIndividually already completed for deposit {}", depositId);
return;
}
if (editFiles.getAddUnrestrictedIndividually().isEmpty()) {
log.debug("No unrestricted files to add individually for deposit {}", depositId);
} else {
addFilesIndividually(editFiles.getAddUnrestrictedIndividually(), false);
}
editFilesLog.getAddUnrestrictedIndividually().setCompleted(true);
}

public void addFilesIndividually(List<String> files, boolean restricted) throws IOException, DataverseException {
if (files.isEmpty()) {
log.debug("No files to add individually for deposit {}, restrict = {}", depositId, restricted);
return;
}
log.debug("Start adding {} files individually for deposit {}, restrict = {}", editFiles.getAddRestrictedIndividually().size(), depositId, restricted);
for (var filepath : files) {
log.debug("Adding restricted file: {}", filepath);
int numberAdded = editFilesLog.getAddRestrictedIndividually().getNumberCompleted();
if (numberAdded > 0) {
log.debug("Resuming adding files from number {}", numberAdded);
}
for (int i = numberAdded; i < files.size(); i++) {
var filepath = files.get(i);
log.debug("Adding file: {}", filepath);
var fileMeta = new FileMeta();
fileMeta.setRestricted(restricted);
String realFilepath = filepath;
var realFilepath = filepath;
// TODO: a bit confusing that autorenamedFiles is part of the cache, although the file looked up here has not been added to the dataset yet.
if (filesInDatasetCache.getAutoRenamedFiles().containsKey(filepath)) {
realFilepath = filesInDatasetCache.getAutoRenamedFiles().get(filepath);
Expand All @@ -218,10 +237,19 @@ public void addFilesIndividually(List<String> files, boolean restricted) throws
fileMeta.setLabel(dataversePath.getLabel());
fileMeta.setDirectoryLabel(dataversePath.getDirectoryLabel());
var fileToUpload = dataDir.resolve(filepath);
if (!Files.exists(fileToUpload)) {
throw new IllegalArgumentException("File to add not found in bag: " + filepath);
}
var addedFileMeta = dataverseService.addFile(pid, fileToUpload, fileMeta);
for (var fm : addedFileMeta.getFiles()) {
filesInDatasetCache.put(fm);
}
if (restricted) {
editFilesLog.getAddRestrictedIndividually().setNumberCompleted(++numberAdded);
} else {
editFilesLog.getAddUnrestrictedIndividually().setNumberCompleted(++numberAdded);
}

}
}

Expand Down
300 changes: 300 additions & 0 deletions ...est/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditorAddFilesIndividuallyTest.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,300 @@
/*
* Copyright (C) 2024 DANS - Data Archiving and Networked Services ([email protected])
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nl.knaw.dans.dvingest.core.bagprocessor;

import nl.knaw.dans.dvingest.YamlBeanAssert;
import nl.knaw.dans.dvingest.core.yaml.EditFilesRoot;
import nl.knaw.dans.dvingest.core.yaml.tasklog.EditFilesLog;
import nl.knaw.dans.lib.dataverse.model.dataset.FileList;
import nl.knaw.dans.lib.dataverse.model.file.FileMeta;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;

import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.UUID;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;

public class FilesEditorAddFilesIndividuallyTest extends FilesEditorTestFixture {
@Test
public void addRestrictedIndividually_adds_files_to_dataset() throws Exception {
// Given
ArgumentCaptor<FileMeta> fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class);
when(dataverseServiceMock.addFile(anyString(), any(Path.class), fileMetaCaptor.capture()))
.thenAnswer(invocation -> new FileList(List.of(fileMetaCaptor.getValue())));
Files.createFile(dataDir.resolve("file1"));
Files.createFile(dataDir.resolve("file2"));
var editFilesRoot = yamlService.readYamlFromString("""
editFiles:
addRestrictedIndividually:
- file1
- file2
""", EditFilesRoot.class);
var editFilesLog = new EditFilesLog();
var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServicesMock, editFilesLog);

// When
filesEditor.editFiles("pid");

// Then
Mockito.verify(dataverseServiceMock).addFile(anyString(), eq(dataDir.resolve("file1")), Mockito.any(FileMeta.class));
Mockito.verify(dataverseServiceMock).addFile(anyString(), eq(dataDir.resolve("file2")), Mockito.any(FileMeta.class));
YamlBeanAssert.assertThat(editFilesLog.getAddRestrictedIndividually()).isEqualTo("""
numberCompleted: 2
completed: true
""");
}

@Test
public void addUnrestrictedFilesIndividually_adds_files_to_dataset() throws Exception {
// Given
ArgumentCaptor<FileMeta> fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class);
when(dataverseServiceMock.addFile(anyString(), any(Path.class), fileMetaCaptor.capture()))
.thenAnswer(invocation -> new FileList(List.of(fileMetaCaptor.getValue())));
Files.createFile(dataDir.resolve("file3"));
Files.createFile(dataDir.resolve("file4"));
var editFilesRoot = yamlService.readYamlFromString("""
editFiles:
addUnrestrictedIndividually:
- file3
- file4
""", EditFilesRoot.class);
var editFilesLog = new EditFilesLog();
var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServicesMock, editFilesLog);

// When
filesEditor.editFiles("pid");

// Then
Mockito.verify(dataverseServiceMock).addFile(anyString(), eq(dataDir.resolve("file3")), Mockito.any(FileMeta.class));
Mockito.verify(dataverseServiceMock).addFile(anyString(), eq(dataDir.resolve("file4")), Mockito.any(FileMeta.class));
YamlBeanAssert.assertThat(editFilesLog.getAddUnrestrictedIndividually()).isEqualTo("""
numberCompleted: 2
completed: true
""");
}

@Test
public void addRestrictedFilesIndividually_throws_exception_when_file_not_found() throws Exception {
// Given
ArgumentCaptor<FileMeta> fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class);
when(dataverseServiceMock.addFile(anyString(), any(Path.class), fileMetaCaptor.capture()))
.thenAnswer(invocation -> new FileList(List.of(fileMetaCaptor.getValue())));
var editFilesRoot = yamlService.readYamlFromString("""
editFiles:
addRestrictedIndividually:
- file1
- file5
""", EditFilesRoot.class);
// Create file1 but not file5
Files.createFile(dataDir.resolve("file1"));
var editFilesLog = new EditFilesLog();
var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServicesMock, editFilesLog);

// When
assertThatThrownBy(() -> filesEditor.editFiles("pid"))
.isInstanceOf(RuntimeException.class)
.hasMessageContaining("File to add not found in bag: file5");
YamlBeanAssert.assertThat(editFilesLog.getAddRestrictedIndividually()).isEqualTo("""
numberCompleted: 1
completed: false
""");
}

@Test
public void addUnrestrictedFilesIndividually_throws_exception_when_file_not_found() throws Exception {
// Given
ArgumentCaptor<FileMeta> fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class);
when(dataverseServiceMock.addFile(anyString(), any(Path.class), fileMetaCaptor.capture()))
.thenAnswer(invocation -> new FileList(List.of(fileMetaCaptor.getValue())));
var editFilesRoot = yamlService.readYamlFromString("""
editFiles:
addUnrestrictedIndividually:
- file3
- file6
""", EditFilesRoot.class);
// create file3 but not file6
Files.createFile(dataDir.resolve("file3"));
var editFilesLog = new EditFilesLog();
var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServicesMock, editFilesLog);

// When
assertThatThrownBy(() -> filesEditor.editFiles("pid"))
.isInstanceOf(RuntimeException.class)
.hasMessageContaining("File to add not found in bag: file6");
YamlBeanAssert.assertThat(editFilesLog.getAddUnrestrictedIndividually()).isEqualTo("""
numberCompleted: 1
completed: false
""");
}

@Test
public void addRestrictedFilesIndividually_is_skipped_when_already_completed() throws Exception {
// Given
var editFilesRoot = yamlService.readYamlFromString("""
editFiles:
addRestrictedIndividually:
- file1
- file2
""", EditFilesRoot.class);
var editFilesLog = new EditFilesLog();
editFilesLog.getAddRestrictedIndividually().setCompleted(true);
editFilesLog.getAddRestrictedIndividually().setNumberCompleted(2);
var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServicesMock, editFilesLog);

// When
filesEditor.addRestrictedFilesIndividually();

// Then
Mockito.verify(dataverseServiceMock, Mockito.never()).addFile(Mockito.anyString(), Mockito.any(Path.class), Mockito.any(FileMeta.class));
YamlBeanAssert.assertThat(editFilesLog.getAddRestrictedIndividually()).isEqualTo("""
numberCompleted: 2
completed: true
""");
}

@Test
public void addUnrestrictedFilesIndividually_is_skipped_when_already_completed() throws Exception {
// Given
var editFilesRoot = yamlService.readYamlFromString("""
editFiles:
addUnrestrictedIndividually:
- file3
- file4
""", EditFilesRoot.class);
var editFilesLog = new EditFilesLog();
editFilesLog.getAddUnrestrictedIndividually().setCompleted(true);
editFilesLog.getAddUnrestrictedIndividually().setNumberCompleted(2);
var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServicesMock, editFilesLog);

// When
filesEditor.addUnrestrictedFilesIndividually();

// Then
Mockito.verify(dataverseServiceMock, Mockito.never()).addFile(Mockito.anyString(), Mockito.any(Path.class), Mockito.any(FileMeta.class));
YamlBeanAssert.assertThat(editFilesLog.getAddUnrestrictedIndividually()).isEqualTo("""
numberCompleted: 2
completed: true
""");
}

@Test
public void addRestrictedFilesIndividually_will_continue_after_number_already_completed() throws Exception {
// Given
ArgumentCaptor<FileMeta> fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class);
when(dataverseServiceMock.addFile(anyString(), any(Path.class), fileMetaCaptor.capture()))
.thenAnswer(invocation -> new FileList(List.of(fileMetaCaptor.getValue())));
Files.createFile(dataDir.resolve("file1"));
Files.createFile(dataDir.resolve("file2"));
Files.createFile(dataDir.resolve("file3"));
var editFilesRoot = yamlService.readYamlFromString("""
editFiles:
addRestrictedIndividually:
- file1
- file2
- file3
""", EditFilesRoot.class);
var editFilesLog = new EditFilesLog();
editFilesLog.getAddRestrictedIndividually().setCompleted(false);
editFilesLog.getAddRestrictedIndividually().setNumberCompleted(2);
var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServicesMock, editFilesLog);

// When
filesEditor.editFiles("pid");

// Then
Mockito.verify(dataverseServiceMock).addFile(anyString(), eq(dataDir.resolve("file3")), Mockito.any(FileMeta.class));
YamlBeanAssert.assertThat(editFilesLog.getAddRestrictedIndividually()).isEqualTo("""
numberCompleted: 3
completed: true
""");
}

// The same for unrestricted
@Test
public void addUnrestrictedFilesIndividually_will_continue_after_number_already_completed() throws Exception {
// Given
ArgumentCaptor<FileMeta> fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class);
when(dataverseServiceMock.addFile(anyString(), any(Path.class), fileMetaCaptor.capture()))
.thenAnswer(invocation -> new FileList(List.of(fileMetaCaptor.getValue())));
Files.createFile(dataDir.resolve("file1"));
Files.createFile(dataDir.resolve("file2"));
Files.createFile(dataDir.resolve("file3"));
var editFilesRoot = yamlService.readYamlFromString("""
editFiles:
addUnrestrictedIndividually:
- file1
- file2
- file3
""", EditFilesRoot.class);
var editFilesLog = new EditFilesLog();
editFilesLog.getAddUnrestrictedIndividually().setCompleted(false);
editFilesLog.getAddUnrestrictedIndividually().setNumberCompleted(2);
var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServicesMock, editFilesLog);

// When
filesEditor.editFiles("pid");

// Then
Mockito.verify(dataverseServiceMock).addFile(anyString(), eq(dataDir.resolve("file3")), Mockito.any(FileMeta.class));
YamlBeanAssert.assertThat(editFilesLog.getAddUnrestrictedIndividually()).isEqualTo("""
numberCompleted: 3
completed: true
""");
}

@Test
public void addRestrictedFilesIndividually_will_auto_rename_file() throws Exception {
// Given
ArgumentCaptor<FileMeta> fileMetaCaptor = ArgumentCaptor.forClass(FileMeta.class);
when(dataverseServiceMock.addFile(anyString(), any(Path.class), fileMetaCaptor.capture()))
.thenAnswer(invocation -> new FileList(List.of(fileMetaCaptor.getValue())));
Files.createFile(dataDir.resolve("file1"));
Files.createFile(dataDir.resolve("file2"));
Files.createFile(dataDir.resolve("file3"));
var editFilesRoot = yamlService.readYamlFromString("""
editFiles:
addRestrictedIndividually:
- file1
- file2
- file3
autoRenameFiles:
- from: file3
to: file3-renamed
""", EditFilesRoot.class);
var editFilesLog = new EditFilesLog();
var filesEditor = new FilesEditor(UUID.randomUUID(), dataDir, editFilesRoot.getEditFiles(), dataverseServiceMock, utilityServicesMock, editFilesLog);
// When
filesEditor.editFiles("pid");

// Then
Mockito.verify(dataverseServiceMock).addFile(eq("pid"), eq(dataDir.resolve("file3")), fileMetaCaptor.capture());
assertThat(fileMetaCaptor.getValue().getLabel()).isEqualTo("file3-renamed");
YamlBeanAssert.assertThat(editFilesLog.getAddRestrictedIndividually()).isEqualTo("""
numberCompleted: 3
completed: true
""");
}

}

0 comments on commit 43b73ab

Please sign in to comment.