diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f58f5b69..624efd97d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,3 +31,7 @@ jobs: run: mvn -V -B -DskipTests=true install -DnvdApiKey=${{ secrets.NVD_API_KEY }} - name: Maven Test run: mvn -B verify -DnvdApiKey=${{ secrets.NVD_API_KEY }} + env: + AWS_REGION: eu-west-2 + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test diff --git a/droid-api/pom.xml b/droid-api/pom.xml index 5a618566c..1959a6bff 100644 --- a/droid-api/pom.xml +++ b/droid-api/pom.xml @@ -75,6 +75,9 @@ org.glassfish.jaxb:jaxb-runtime javax.xml.bind:jaxb-api:jar + + software.amazon.awssdk:sdk-core:jar + commons-lang:commons-lang:jar @@ -100,7 +103,15 @@ - + + org.apache.maven.plugins + maven-compiler-plugin + + 21 + 21 + + + @@ -130,10 +141,59 @@ jaxb-runtime - junit - junit + org.junit.jupiter + junit-jupiter-params + ${junit.version} test + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + + + software.amazon.awssdk + s3 + ${aws.version} + + + software.amazon.awssdk + regions + ${aws.version} + + + software.amazon.awssdk + aws-core + ${aws.version} + + + software.amazon.awssdk + sdk-core + ${aws.version} + + + org.apache.httpcomponents + httpcore + 4.4.16 + test + + + org.apache.commons + commons-lang3 + 3.17.0 + test + + + org.apache.httpcomponents + httpclient + 4.5.14 + + + org.hamcrest + hamcrest + test + jakarta.xml.bind jakarta.xml.bind-api @@ -150,10 +210,5 @@ 5.2.5 test - - org.hamcrest - hamcrest - test - diff --git a/droid-api/src/main/java/uk/gov/nationalarchives/droid/internal/api/ApiResult.java b/droid-api/src/main/java/uk/gov/nationalarchives/droid/internal/api/ApiResult.java index 51f2faf17..f06424780 100644 --- a/droid-api/src/main/java/uk/gov/nationalarchives/droid/internal/api/ApiResult.java +++ b/droid-api/src/main/java/uk/gov/nationalarchives/droid/internal/api/ApiResult.java @@ -33,19 +33,23 @@ import uk.gov.nationalarchives.droid.core.interfaces.IdentificationMethod; +import java.net.URI; + public class ApiResult { private final String extension; private final IdentificationMethod method; private final String puid; private final String name; private final boolean fileExtensionMismatch; + private final URI uri; - public ApiResult(String extension, IdentificationMethod method, String puid, String name, boolean fileExtensionMismatch) { + public ApiResult(String extension, IdentificationMethod method, String puid, String name, boolean fileExtensionMismatch, URI uri) { this.extension = extension; this.method = method; this.puid = puid; this.name = name; this.fileExtensionMismatch = fileExtensionMismatch; + this.uri = uri; } public String getName() { @@ -67,4 +71,8 @@ public String getExtension() { public boolean isFileExtensionMismatch() { return fileExtensionMismatch; } + + public URI getUri() { + return uri; + } } diff --git a/droid-api/src/main/java/uk/gov/nationalarchives/droid/internal/api/DroidAPI.java b/droid-api/src/main/java/uk/gov/nationalarchives/droid/internal/api/DroidAPI.java index 5c5c7fa30..384c09766 100644 --- a/droid-api/src/main/java/uk/gov/nationalarchives/droid/internal/api/DroidAPI.java +++ b/droid-api/src/main/java/uk/gov/nationalarchives/droid/internal/api/DroidAPI.java @@ -31,9 +31,13 @@ */ package uk.gov.nationalarchives.droid.internal.api; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.http.HttpClient; import java.nio.file.Files; import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -43,16 +47,19 @@ import org.apache.commons.lang.StringUtils; +import org.apache.http.client.utils.URIBuilder; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Uri; +import software.amazon.awssdk.services.s3.S3Utilities; +import software.amazon.awssdk.services.s3.model.S3Object; import uk.gov.nationalarchives.droid.core.BinarySignatureIdentifier; import uk.gov.nationalarchives.droid.core.SignatureParseException; -import uk.gov.nationalarchives.droid.core.interfaces.DroidCore; -import uk.gov.nationalarchives.droid.core.interfaces.IdentificationMethod; -import uk.gov.nationalarchives.droid.core.interfaces.IdentificationResultCollection; -import uk.gov.nationalarchives.droid.core.interfaces.IdentificationResult; -import uk.gov.nationalarchives.droid.core.interfaces.RequestIdentifier; +import uk.gov.nationalarchives.droid.core.interfaces.*; import uk.gov.nationalarchives.droid.core.interfaces.archive.ContainerIdentifier; -import uk.gov.nationalarchives.droid.core.interfaces.resource.FileSystemIdentificationRequest; -import uk.gov.nationalarchives.droid.core.interfaces.resource.RequestMetaData; +import uk.gov.nationalarchives.droid.core.interfaces.resource.*; /** @@ -60,20 +67,22 @@ * TNA INTERNAL !!! class which encapsulate DROID internal non-friendly api and expose it in simple way. *

*

- * To obtain instance of this class, use factory method {@link #getInstance(Path, Path)} to obtain instance. + * To obtain instance of this class, use the DroidAPIBuilder class to obtain an instance. * Obtaining instance is expensive operation and if used multiple time, instance should be cached. * Instance should be thread-safe, but we didn't run any internal audit. We suggest creating one instance for every thread. *

*

- * To identify file, use method {@link #submit(Path)}. This method take full path to file which should be identified. + * To identify file, use method {@link #submit(URI)}. This method take full uri to file which should be identified. + * The URI can point to either an s3, http, https or file URI. * It returns identification result which can contain 0..N signatures. Bear in mind that single file can have zero to multiple * signature matches! *

*/ -public final class DroidAPI { +public final class DroidAPI implements AutoCloseable { private static final String ZIP_PUID = "x-fmt/263"; private static final String OLE2_PUID = "fmt/111"; + private static final String S3_SCHEME = "s3"; private static final String GZIP_PUID = "x-fmt/266"; private static final AtomicLong ID_GENERATOR = new AtomicLong(); @@ -92,7 +101,14 @@ public final class DroidAPI { private final String droidVersion; - private DroidAPI(DroidCore droidCore, ContainerIdentifier zipIdentifier, ContainerIdentifier ole2Identifier, ContainerIdentifier gzIdentifier, String containerSignatureVersion, String binarySignatureVersion, String droidVersion) { + private final S3Client s3Client; + + private final Region s3Region; + + private final HttpClient httpClient; + + + private DroidAPI(DroidCore droidCore, ContainerIdentifier zipIdentifier, ContainerIdentifier ole2Identifier, ContainerIdentifier gzIdentifier, String containerSignatureVersion, String binarySignatureVersion, String droidVersion, S3Client s3Client, HttpClient httpClient, Region s3Region) { this.droidCore = droidCore; this.zipIdentifier = zipIdentifier; this.ole2Identifier = ole2Identifier; @@ -100,85 +116,223 @@ private DroidAPI(DroidCore droidCore, ContainerIdentifier zipIdentifier, Contain this.containerSignatureVersion = containerSignatureVersion; this.binarySignatureVersion = binarySignatureVersion; this.droidVersion = droidVersion; + this.s3Region = getRegionOrDefault(s3Region); + this.s3Client = getS3ClientOrDefault(s3Client); + this.httpClient = getHttpClientOrDefault(httpClient); } - /** - * Return instance, or throw error. - * @param binarySignature Path to xml file with binary signatures. - * @param containerSignature Path to xml file with contained signatures. - * @return Instance of droid with binary and container signature. - * @throws SignatureParseException On invalid signature file. - */ - public static DroidAPI getInstance(final Path binarySignature, final Path containerSignature) throws SignatureParseException { - BinarySignatureIdentifier droidCore = new BinarySignatureIdentifier(); - droidCore.setSignatureFile(binarySignature.toAbsolutePath().toString()); + private HttpClient getHttpClientOrDefault(HttpClient httpClient) { + if (httpClient == null) { + return HttpClient.newHttpClient(); + } + return httpClient; + } + + private S3Client getS3ClientOrDefault(S3Client s3Client) { + if (s3Client == null) { + return S3Client.builder().region(this.s3Region).build(); + } + return s3Client; + } + + private Region getRegionOrDefault(Region region) { + if (region == null) { + try { + return DefaultAwsRegionProviderChain.builder().build().getRegion(); + } catch (SdkClientException e) { + return Region.EU_WEST_2; + } + } + return region; + } + + @Override + public void close() { + this.httpClient.close(); + this.s3Client.close(); + } + + public static class DroidAPIBuilder { + private Path binarySignature; + private Path containerSignature; + private S3Client s3Client; + private Region s3Region; + private HttpClient httpClient; - droidCore.init(); - droidCore.setMaxBytesToScan(Long.MAX_VALUE); - droidCore.getSigFile().prepareForUse(); - String containerVersion = StringUtils.substringAfterLast(containerSignature.getFileName().toString(), "-").split("\\.")[0]; - String droidVersion = ResourceBundle.getBundle("options").getString("version_no"); - ContainerApi containerApi = new ContainerApi(droidCore, containerSignature); - return new DroidAPI(droidCore, containerApi.zipIdentifier(), containerApi.ole2Identifier(), containerApi.gzIdentifier(), containerVersion, droidCore.getSigFile().getVersion(), droidVersion); + public DroidAPIBuilder binarySignature(final Path binarySignature) { + this.binarySignature = binarySignature; + return this; + } + + public DroidAPIBuilder containerSignature(final Path containerSignature) { + this.containerSignature = containerSignature; + return this; + } + + public DroidAPIBuilder s3Client(final S3Client s3Client) { + this.s3Client = s3Client; + return this; + } + + public DroidAPIBuilder s3Region(final Region s3Region) { + this.s3Region = s3Region; + return this; + } + + public DroidAPIBuilder httpClient(final HttpClient httpClient) { + this.httpClient = httpClient; + return this; + } + + public DroidAPI build() throws SignatureParseException { + if (this.binarySignature == null || this.containerSignature == null) { + throw new IllegalArgumentException("Container signature and binary signature are mandatory arguments"); + } + BinarySignatureIdentifier droidCore = new BinarySignatureIdentifier(); + droidCore.setSignatureFile(binarySignature.toAbsolutePath().toString()); + droidCore.init(); + droidCore.setMaxBytesToScan(Long.MAX_VALUE); + droidCore.getSigFile().prepareForUse(); + String containerVersion = StringUtils.substringAfterLast(containerSignature.getFileName().toString(), "-").split("\\.")[0]; + String droidVersion = ResourceBundle.getBundle("options").getString("version_no"); + ContainerApi containerApi = new ContainerApi(droidCore, containerSignature); + return new DroidAPI(droidCore, containerApi.zipIdentifier(), containerApi.ole2Identifier(), containerApi.gzIdentifier(), containerVersion, droidCore.getSigFile().getVersion(), droidVersion, this.s3Client, this.httpClient, this.s3Region); + } + } + + public static DroidAPIBuilder builder() { + return new DroidAPIBuilder(); } /** * Submit file for identification. It's important that file has proper file extension. If file * can't be identified via binary or container signature, then we use file extension for identification. - * @param file Full path to file for identification. + * @param uri Full URI of the file for identification. * @return File identification result. File can have multiple matching signatures. * @throws IOException If File can't be read or there is IO error. */ - public List submit(final Path file) throws IOException { + public List submit(final URI uri) throws IOException { + if (S3_SCHEME.equals(uri.getScheme())) { + return submitS3Identification(uri); + } else if (List.of("http", "https").contains(uri.getScheme())) { + return submitHttpIdentification(uri); + } else { + return submitFileSystemIdentification(Path.of(uri)); + } + } + + private List submitHttpIdentification(final URI uri) throws IOException { + HttpClient httpClient = this.httpClient == null ? HttpClient.newHttpClient() : this.httpClient; + HttpUtils httpUtils = new HttpUtils(httpClient); + HttpUtils.HttpMetadata httpMetadata = httpUtils.getHttpMetadata(uri); + Long fileSize = httpMetadata.fileSize(); + Long lastModified = httpMetadata.lastModified(); + final RequestMetaData metaData = new RequestMetaData( - Files.size(file), - Files.getLastModifiedTime(file).toMillis(), - file.toAbsolutePath().toString() + fileSize, + lastModified, + uri.toString() ); - final RequestIdentifier id = new RequestIdentifier(file.toAbsolutePath().toUri()); + final RequestIdentifier id = getRequestIdentifier(uri); + + + try (final HttpIdentificationRequest request = new HttpIdentificationRequest(metaData, id, httpClient)) { + request.open(uri); + return getApiResults(request); + } + } + + private List submitS3Identification(final URI uri) throws IOException { + Region region = this.s3Region == null ? DefaultAwsRegionProviderChain.builder().build().getRegion() : this.s3Region; + S3Client client; + if (this.s3Client == null) { + client = S3Client.builder().region(region).build(); + } else { + client = this.s3Client; + } + S3Utils s3Utils = new S3Utils(client); + S3Utils.S3ObjectList objectList = s3Utils.listObjects(uri); + List apiResults = new ArrayList<>(); + + for (S3Object s3Object: objectList.contents()) { + URIBuilder uriBuilder = new URIBuilder(); + URI objectUri; + try { + objectUri = uriBuilder.setScheme(S3_SCHEME).setHost(objectList.bucket()).setPath(s3Object.key()).build(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + S3Uri s3Uri = S3Utilities.builder().region(region).build().parseUri(objectUri); + final RequestIdentifier id = getRequestIdentifier(s3Uri.uri()); + RequestMetaData metaData = new RequestMetaData(s3Object.size(), s3Object.lastModified().getEpochSecond(), s3Uri.uri().toString()); + try (final S3IdentificationRequest request = new S3IdentificationRequest(metaData, id, client)) { + request.open(s3Uri); + apiResults.addAll(getApiResults(request)); + } + } + return apiResults; + } + + private static RequestIdentifier getRequestIdentifier(URI uri) { + final RequestIdentifier id = new RequestIdentifier(uri); id.setParentId(ID_GENERATOR.getAndIncrement()); id.setNodeId(ID_GENERATOR.getAndIncrement()); + return id; + } - IdentificationResultCollection resultCollection; + + private List submitFileSystemIdentification(final Path file) throws IOException { + final RequestMetaData metaData = new RequestMetaData( + Files.size(file), + Files.getLastModifiedTime(file).toMillis(), + file.toAbsolutePath().toString() + ); + + final RequestIdentifier id = getRequestIdentifier(file.toAbsolutePath().toUri()); try (final FileSystemIdentificationRequest request = new FileSystemIdentificationRequest(metaData, id)) { request.open(file); - String extension = request.getExtension(); - - IdentificationResultCollection binaryResult = droidCore.matchBinarySignatures(request); - Optional containerPuid = getContainerPuid(binaryResult); + return getApiResults(request); + } + } - if (containerPuid.isPresent()) { - resultCollection = handleContainer(binaryResult, request, containerPuid.get()); + private List getApiResults(IdentificationRequest request) throws IOException { + IdentificationResultCollection resultCollection; + String extension = request.getExtension(); + + IdentificationResultCollection binaryResult = droidCore.matchBinarySignatures(request); + Optional containerPuid = getContainerPuid(binaryResult); + + if (containerPuid.isPresent()) { + resultCollection = handleContainer(binaryResult, request, containerPuid.get()); + } else { + droidCore.removeLowerPriorityHits(binaryResult); + droidCore.checkForExtensionsMismatches(binaryResult, request.getExtension()); + if (binaryResult.getResults().isEmpty()) { + resultCollection = identifyByExtension(request); } else { - droidCore.removeLowerPriorityHits(binaryResult); - droidCore.checkForExtensionsMismatches(binaryResult, request.getExtension()); - if (binaryResult.getResults().isEmpty()) { - resultCollection = identifyByExtension(request); - } else { - resultCollection = binaryResult; - } + resultCollection = binaryResult; } + } - boolean fileExtensionMismatch = resultCollection.getExtensionMismatch(); + boolean fileExtensionMismatch = resultCollection.getExtensionMismatch(); - return resultCollection.getResults() - .stream().map(res -> createApiResult(res, extension, fileExtensionMismatch)) - .collect(Collectors.toList()); - } + return resultCollection.getResults() + .stream().map(res -> createApiResult(res, extension, fileExtensionMismatch, request.getIdentifier().getUri())) + .collect(Collectors.toList()); } - private ApiResult createApiResult(IdentificationResult result, String extension, boolean extensionMismatch) { + private ApiResult createApiResult(IdentificationResult result, String extension, boolean extensionMismatch, URI uri) { String name = result.getName(); if (result.getMethod().equals(IdentificationMethod.CONTAINER) && (droidCore.formatNameByPuid(result.getPuid()) != null)) { name = droidCore.formatNameByPuid(result.getPuid()); } - return new ApiResult(extension, result.getMethod(), result.getPuid(), name, extensionMismatch); + return new ApiResult(extension, result.getMethod(), result.getPuid(), name, extensionMismatch, uri); } - private IdentificationResultCollection identifyByExtension(final FileSystemIdentificationRequest identificationRequest) { + private IdentificationResultCollection identifyByExtension(final IdentificationRequest identificationRequest) { IdentificationResultCollection extensionResult = droidCore.matchExtensions(identificationRequest, false); droidCore.removeLowerPriorityHits(extensionResult); return extensionResult; @@ -191,8 +345,8 @@ private Optional getContainerPuid(final IdentificationResultCollection b .filter(containerPuids::contains).findFirst(); } - private IdentificationResultCollection handleContainer(final IdentificationResultCollection binaryResult, - final FileSystemIdentificationRequest identificationRequest, final String containerPuid) throws IOException { + private IdentificationResultCollection handleContainer(final IdentificationResultCollection binaryResult, + final IdentificationRequest identificationRequest, final String containerPuid) throws IOException { ContainerIdentifier identifier = switch (containerPuid) { case ZIP_PUID -> zipIdentifier; case OLE2_PUID -> ole2Identifier; @@ -220,4 +374,17 @@ public String getBinarySignatureVersion() { public String getDroidVersion() { return droidVersion; } + + public S3Client getS3Client() { + return s3Client; + } + + public HttpClient getHttpClient() { + return httpClient; + } + + public Region getS3Region() { + return s3Region; + } + } diff --git a/droid-api/src/main/resources/junit-platform.properties b/droid-api/src/main/resources/junit-platform.properties new file mode 100644 index 000000000..b10b0e321 --- /dev/null +++ b/droid-api/src/main/resources/junit-platform.properties @@ -0,0 +1 @@ +junit.jupiter.execution.parallel.enabled = true \ No newline at end of file diff --git a/droid-api/src/test/java/uk/gov/nationalarchives/droid/internal/api/DroidAPISkeletonTest.java b/droid-api/src/test/java/uk/gov/nationalarchives/droid/internal/api/DroidAPISkeletonTest.java index 3b40a1063..d710dcbf3 100644 --- a/droid-api/src/test/java/uk/gov/nationalarchives/droid/internal/api/DroidAPISkeletonTest.java +++ b/droid-api/src/test/java/uk/gov/nationalarchives/droid/internal/api/DroidAPISkeletonTest.java @@ -31,17 +31,19 @@ */ package uk.gov.nationalarchives.droid.internal.api; +import com.sun.net.httpserver.HttpServer; import org.hamcrest.Description; import org.hamcrest.TypeSafeMatcher; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import uk.gov.nationalarchives.droid.core.SignatureParseException; import java.io.IOException; +import java.net.URI; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; import java.util.regex.Matcher; @@ -51,6 +53,8 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.*; +import static uk.gov.nationalarchives.droid.internal.api.DroidAPITestUtils.createHttpServer; +import static uk.gov.nationalarchives.droid.internal.api.DroidAPITestUtils.createS3Server; /** * Test internal API against skeleton sample. Unfortunately skeleton sample is in different project, @@ -58,37 +62,48 @@ * The test looks at puid mentioned in the filename and expects that as a result from the API * */ -@RunWith(Parameterized.class) public class DroidAPISkeletonTest { - private final String puid; - private final Path path; - private final DroidAPI api; + private static HttpServer s3Server; - public DroidAPISkeletonTest(String puid, Path path, DroidAPI api) { - this.puid = puid; - this.path = path; - this.api = api; + private static HttpServer httpServer; + + private static DroidAPI api; + + @BeforeAll + static void setup() throws IOException, SignatureParseException { + s3Server = createS3Server(); + httpServer = createHttpServer(); + api = DroidAPITestUtils.createApi(URI.create("http://localhost:" + s3Server.getAddress().getPort())); } - @Parameters (name = "Testing file \"{1}\" for format \"{0}\"") - public static Collection data() throws IOException, SignatureParseException { - Pattern FILENAME = Pattern.compile("((?:x-)?fmt)-(\\d+)-signature-id-(\\d+).*"); + public record SkeletonTest(String puid, URI uri) {} + public static Stream data() throws IOException, SignatureParseException { + Pattern FILENAME = Pattern.compile("((?:x-)?fmt)-(\\d+)-signature-id-(\\d+).*"); Set ignorePuid = getIgnoredPuids(); - DroidAPI api = DroidAPITestUtils.createApi(); return Stream.concat( Files.list(Paths.get("../droid-core/test-skeletons/fmt")), Files.list(Paths.get("../droid-core/test-skeletons/x-fmt")) - ).map(x -> { + ).flatMap(x -> { Matcher z = FILENAME.matcher(x.getFileName().toString()); if (!z.matches()) { return null; } else { - return new Object[]{z.group(1) + "/" + z.group(2), x, api}; + String puid = z.group(1) + "/" + z.group(2); + if (ignorePuid.contains(puid)) { + return null; + } + String uriPath = x.toUri().getPath() + .replaceAll(" ", "%20") + .replaceAll("\\\\", "/"); + return Stream.of( + new SkeletonTest(puid, x.toUri()), + new SkeletonTest(puid, URI.create("s3://localhost:" + s3Server.getAddress().getPort() + uriPath)), + new SkeletonTest(puid, URI.create("s3://localhost:" + httpServer.getAddress().getPort() + uriPath)) + ); } - }).filter(x -> x != null && !ignorePuid.contains(x[0].toString())) - .collect(Collectors.toList()); + }).filter(Objects::nonNull); } /** @@ -111,10 +126,12 @@ private static Set getIgnoredPuids() { } - @Test - public void skeletonTest() throws Exception { - List results = api.submit(path); - assertThat(results, hasItem(ResultMatcher.resultWithPuid(puid))); + @Execution(ExecutionMode.CONCURRENT) + @ParameterizedTest + @MethodSource("data") + public void skeletonTest(SkeletonTest skeletonTest) throws Exception { + List results = api.submit(skeletonTest.uri); + assertThat(results, hasItem(ResultMatcher.resultWithPuid(skeletonTest.puid))); } private static class ResultMatcher extends TypeSafeMatcher { diff --git a/droid-api/src/test/java/uk/gov/nationalarchives/droid/internal/api/DroidAPITest.java b/droid-api/src/test/java/uk/gov/nationalarchives/droid/internal/api/DroidAPITest.java index ee4957d9a..935475448 100644 --- a/droid-api/src/test/java/uk/gov/nationalarchives/droid/internal/api/DroidAPITest.java +++ b/droid-api/src/test/java/uk/gov/nationalarchives/droid/internal/api/DroidAPITest.java @@ -31,119 +31,149 @@ */ package uk.gov.nationalarchives.droid.internal.api; -import org.junit.Before; -import org.junit.Test; +import com.sun.net.httpserver.HttpServer; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; import uk.gov.nationalarchives.droid.core.SignatureParseException; import uk.gov.nationalarchives.droid.core.interfaces.IdentificationMethod; import uk.gov.nationalarchives.droid.internal.api.DroidAPITestUtils.ContainerType; import java.io.IOException; -import java.nio.file.Path; +import java.net.URI; +import java.net.http.HttpClient; import java.nio.file.Paths; +import java.nio.file.Path; import java.util.List; import java.util.Optional; import java.util.ResourceBundle; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import static org.junit.jupiter.api.Assertions.*; import static uk.gov.nationalarchives.droid.internal.api.DroidAPITestUtils.*; public class DroidAPITest { - private DroidAPI api; + private static final String DATA = "TEST"; + + private static DroidAPI api; + private static HttpServer s3Server; + private static HttpServer httpServer; + private static URI endpointOverride; + + private static Stream getUris(String path) { + URI fileUri = Paths.get(path).toUri(); + URI s3Uri = URI.create("s3://127.0.0.1" + ":" + s3Server.getAddress().getPort() + fileUri.getPath()); + URI httpUri = URI.create("http://127.0.0.1" + ":" + httpServer.getAddress().getPort() + fileUri.getPath()); + return Stream.of(fileUri, s3Uri, httpUri); + } - @Before - public void setup() throws SignatureParseException { - api = DroidAPITestUtils.createApi(); + @BeforeAll + public static void setup() throws SignatureParseException, IOException { + s3Server = createS3Server(); + httpServer = createHttpServer(); + endpointOverride = URI.create("http://127.0.0.1" + ":" + s3Server.getAddress().getPort()); + api = DroidAPITestUtils.createApi(endpointOverride); } + + @Test public void should_create_non_null_instance_using_test_utility_class() { assertThat(api, is(notNullValue())); } - @Test - public void should_match_gzip_container_file() { - String data = "TEST"; - ContainerType containerType = new ContainerType("GZIP", generateId(),"x-fmt/266"); - DroidAPI api = DroidAPITestUtils.createApiForContainer(new DroidAPITestUtils.ContainerFile(containerType, data, "fmt/12345", Optional.empty())); - try { - List results = api.submit(DroidAPITestUtils.generateGzFile(data)); - assertThat(results, hasSize(1)); - assertThat(results.getFirst().getPuid(), is("fmt/12345")); - assertThat(results.getFirst().getMethod(), is(IdentificationMethod.CONTAINER)); - } catch (IOException e) { - throw new RuntimeException(e); - } + public record ContainerTest(URI uri, ContainerType containerType, Optional path) {} + + static Stream signatureTests() { + ContainerType gzipContainerType = new ContainerType("GZIP", generateId(),"x-fmt/266"); + ContainerType zipContainerType = new ContainerType("ZIP", generateId(),"x-fmt/263"); + ContainerType ole2ContainerType = new ContainerType("OLE2", generateId(),"fmt/111"); + Stream gzipStream = getUris(generateGzFile(DATA).toString()).map(uri -> new ContainerTest(uri, gzipContainerType, Optional.empty())); + Stream zipStream = getUris(generateZipFile(DATA, DATA).toString()).map(uri -> new ContainerTest(uri, zipContainerType, Optional.of(DATA))); + Stream ole2Stream = getUris(generateOle2File(DATA, DATA).toString()).map(uri -> new ContainerTest(uri, ole2ContainerType, Optional.of(DATA))); + return Stream.concat(Stream.concat(gzipStream, zipStream), ole2Stream); } - @Test - public void should_match_zip_container_file() { - String data = "TEST"; - ContainerType containerType = new ContainerType("ZIP", generateId(),"x-fmt/263"); - DroidAPI api = DroidAPITestUtils.createApiForContainer(new DroidAPITestUtils.ContainerFile(containerType, data, "fmt/12345", Optional.of(data))); - try { - List results = api.submit(DroidAPITestUtils.generateZipFile(data, data)); + + @ParameterizedTest + @MethodSource("signatureTests") + public void should_match_container_files(ContainerTest containerTest) throws IOException { + ContainerFile containerFile = new ContainerFile(containerTest.containerType, DATA, "fmt/12345", containerTest.path); + try (DroidAPI api = DroidAPITestUtils.createApiForContainer(endpointOverride, containerFile)) { + List results = api.submit(containerTest.uri); assertThat(results, hasSize(1)); assertThat(results.getFirst().getPuid(), is("fmt/12345")); assertThat(results.getFirst().getMethod(), is(IdentificationMethod.CONTAINER)); - } catch (IOException e) { - throw new RuntimeException(e); } } @Test - public void should_match_ole2_container_file() { - String data = "TEST"; - ContainerType containerType = new ContainerType("OLE2", generateId(),"fmt/111"); - DroidAPI api = DroidAPITestUtils.createApiForContainer(new DroidAPITestUtils.ContainerFile(containerType, data, "fmt/12345", Optional.of(data))); - try { - List results = api.submit(DroidAPITestUtils.generateOle2File(data, data)); - assertThat(results, hasSize(1)); - assertThat(results.getFirst().getPuid(), is("fmt/12345")); - assertThat(results.getFirst().getMethod(), is(IdentificationMethod.CONTAINER)); - } catch (IOException e) { - throw new RuntimeException(e); - } + public void should_throw_an_exception_if_file_cannot_be_read() { + assertThrows(IOException.class, () -> api.submit(Path.of("/invalidpath").toUri())); } - @Test(expected = IOException.class) - public void should_throw_an_exception_if_file_cannot_be_read() throws IOException { - api.submit(Path.of("/invalidpath")); + @Test + public void should_throw_an_exception_if_container_file_cannot_be_read() { + assertThrows(RuntimeException.class, () -> { + DroidAPI.builder() + .binarySignature(signaturePath) + .containerSignature(Path.of("/invalidContainerPath")) + .build(); + }); } - @Test(expected = RuntimeException.class) - public void should_throw_an_exception_if_container_file_cannot_be_read() throws SignatureParseException { - DroidAPI.getInstance(signaturePath, Path.of("/invalidContainerPath")); + @Test + public void should_throw_an_exception_if_signature_file_cannot_be_read() { + assertThrows(SignatureParseException.class, () -> { + DroidAPI.builder() + .binarySignature(Path.of("/invalidSignaturePath")) + .containerSignature(containerPath) + .build(); + }); } - @Test(expected = SignatureParseException.class) - public void should_throw_an_exception_if_signature_file_cannot_be_read() throws SignatureParseException { - DroidAPI.getInstance(Path.of("/invalidSignaturePath"), containerPath); + static Stream binarySignatureUris() { + return getUris("src/test/resources/persistence.zip"); } - @Test - public void should_identify_given_file_with_binary_signature() throws IOException { - List results = api.submit( - Paths.get("src/test/resources/persistence.zip")); + @Execution(ExecutionMode.CONCURRENT) + @ParameterizedTest + @MethodSource("binarySignatureUris") + public void should_identify_given_file_with_binary_signature(URI uri) throws IOException { + List results = api.submit(uri); assertThat(results, is(notNullValue())); assertThat(results.size(), is(1)); - ApiResult identificationResult = results.getFirst(); + ApiResult identificationResult = results.get(0); assertThat(identificationResult.getPuid(), is("x-fmt/263")); assertThat(identificationResult.getName(), is("ZIP Format")); assertThat(identificationResult.getMethod(), is(IdentificationMethod.BINARY_SIGNATURE)); + } - @Test - public void should_identify_given_file_using_container_signature() throws IOException { - List results = api.submit( - Paths.get("../droid-container/src/test/resources/odf_text.odt")); + static Stream containerSignatureUris() { + return getUris("../droid-container/src/test/resources/odf_text.odt"); + } + + @Execution(ExecutionMode.CONCURRENT) + @ParameterizedTest + @MethodSource("containerSignatureUris") + public void should_identify_given_file_using_container_signature(URI uri) throws IOException { + List results = api.submit(uri); assertThat(results, is(notNullValue())); assertThat(results.size(), is(1)); @@ -155,9 +185,15 @@ public void should_identify_given_file_using_container_signature() throws IOExce assertThat(identificationResult.getMethod(), is(IdentificationMethod.CONTAINER)); } - @Test - public void should_identify_given_file_using_file_extension() throws IOException { - List results = api.submit(Paths.get("src/test/resources/test.txt")); + static Stream fileExtensionUris() { + return getUris("src/test/resources/test.txt"); + } + + @Execution(ExecutionMode.CONCURRENT) + @ParameterizedTest + @MethodSource("fileExtensionUris") + public void should_identify_given_file_using_file_extension(URI uri) throws IOException { + List results = api.submit(uri); assertThat(results, is(notNullValue())); assertThat(results, hasSize(1)); @@ -167,18 +203,35 @@ public void should_identify_given_file_using_file_extension() throws IOException assertThat(singleResult.getMethod(), is(IdentificationMethod.EXTENSION)); } - @Test - public void should_report_extension_of_the_file_under_identification_test() throws IOException { - List resultsWithExtension = api.submit(Paths.get("src/test/resources/test.txt")); - List resultsWithoutExtension = api.submit(Paths.get("src/test/resources/word97")); + static Stream> correctExtensionUris() { + List withExtensionList = getUris("src/test/resources/test.txt").toList(); + List withoutExtensionList = getUris("src/test/resources/word97").toList(); + return Stream.of( + Pair.of(withExtensionList.getFirst(), withoutExtensionList.getFirst()), + Pair.of(withExtensionList.getLast(), withoutExtensionList.getLast()) + ); + } + + @Execution(ExecutionMode.CONCURRENT) + @ParameterizedTest + @MethodSource("correctExtensionUris") + public void should_report_extension_of_the_file_under_identification_test(Pair uriPair) throws IOException { + List resultsWithExtension = api.submit(uriPair.getLeft()); + List resultsWithoutExtension = api.submit(uriPair.getRight()); assertThat(resultsWithExtension.getFirst().getExtension(), is("txt")); assertThat(resultsWithoutExtension.getFirst().getExtension(), is("")); } - @Test - public void should_report_all_puids_when_there_are_more_than_one_identification_hits() throws IOException { - List results = api.submit(Paths.get("src/test/resources/double-identification.jpg")); + static Stream doubleIdentificationUris() { + return getUris("src/test/resources/double-identification.jpg"); + } + + @Execution(ExecutionMode.CONCURRENT) + @ParameterizedTest + @MethodSource("doubleIdentificationUris") + public void should_report_all_puids_when_there_are_more_than_one_identification_hits(URI uri) throws IOException { + List results = api.submit(uri); assertThat(results.size(), is(2)); assertThat(results.stream().map(ApiResult::getPuid).collect(Collectors.toList()), containsInAnyOrder("fmt/96", "fmt/41")); @@ -186,9 +239,15 @@ public void should_report_all_puids_when_there_are_more_than_one_identification_ containsInAnyOrder("Raw JPEG Stream", "Hypertext Markup Language")); } - @Test + static Stream extensionMismatchUris() { + return getUris("src/test/resources/docx-file-as-xls.xlsx"); + } + + @Execution(ExecutionMode.CONCURRENT) + @ParameterizedTest + @MethodSource("extensionMismatchUris") public void should_report_when_there_is_an_extension_mismatch() throws IOException { - List results = api.submit(Paths.get("src/test/resources/docx-file-as-xls.xlsx")); + List results = api.submit(Paths.get("src/test/resources/docx-file-as-xls.xlsx").toUri()); assertThat(results.size(), is(1)); assertThat(results.getFirst().getPuid(), is("fmt/412")); assertThat(results.getFirst().isFileExtensionMismatch(), is(true)); @@ -201,28 +260,72 @@ public void should_report_correct_version_for_the_binary_and_container_signature assertThat(api.getBinarySignatureVersion(), is("119")); } + static Stream emptyFileUris() { + return getUris("src/test/resources/test"); + } + @Test public void should_produce_zero_results_for_an_empty_file() throws IOException { - List results = api.submit(Paths.get("src/test/resources/test")); + List results = api.submit(Paths.get("src/test/resources/test").toUri()); assertThat(results, hasSize(0)); } - @Test + @Execution(ExecutionMode.CONCURRENT) + @ParameterizedTest + @MethodSource("emptyFileUris") public void should_produce_results_for_every_time_a_file_is_submitted_for_identification() throws IOException { final int MAX_ITER = 5000; int acc = 0; for (int i = 0; i < MAX_ITER; i++) { List results = api.submit( - Paths.get("../droid-container/src/test/resources/odf_text.odt")); + Paths.get("../droid-container/src/test/resources/odf_text.odt").toUri()); acc += results.size(); } assertThat(acc, is(MAX_ITER)); } - @Test + static Stream fmtFortyUris() { + return getUris("../droid-container/src/test/resources/word97.doc"); + } + + @Execution(ExecutionMode.CONCURRENT) + @ParameterizedTest + @MethodSource("fmtFortyUris") public void should_identify_fmt_40_correctly_with_container_identification_method() throws IOException { List results = api.submit( - Paths.get("../droid-container/src/test/resources/word97.doc")); + Paths.get("../droid-container/src/test/resources/word97.doc").toUri()); assertThat(results.getFirst().getName(), is("Microsoft Word Document")); } + + @Test + public void should_return_an_error_if_signature_paths_are_not_set() { + assertThrows(IllegalArgumentException.class, () -> DroidAPI.builder().build()); + assertThrows(IllegalArgumentException.class, () -> DroidAPI.builder().containerSignature(containerPath).build()); + assertThrows(IllegalArgumentException.class, () -> DroidAPI.builder().binarySignature(signaturePath).build()); + } + + @Test + public void should_provide_default_clients_if_none_are_provided() throws SignatureParseException { + DroidAPI.builder().binarySignature(signaturePath).containerSignature(containerPath).build(); + assertNotNull(api.getS3Client()); + assertNotNull(api.getHttpClient()); + } + + @Test + public void should_default_to_london_region_if_no_region_provided() throws SignatureParseException { + DroidAPI api = DroidAPI.builder().binarySignature(signaturePath).containerSignature(containerPath).build(); + assertEquals(api.getS3Region(), Region.EU_WEST_2); + } + + @Test + public void should_close_clients_after_use() throws SignatureParseException { + S3Client s3Client; + HttpClient httpClient; + try (DroidAPI api = DroidAPI.builder().binarySignature(signaturePath).containerSignature(containerPath).build()) { + httpClient = api.getHttpClient(); + s3Client = api.getS3Client(); + } + assertTrue(httpClient.isTerminated()); + assertThrows(IllegalStateException.class, s3Client::listBuckets); + } } diff --git a/droid-api/src/test/java/uk/gov/nationalarchives/droid/internal/api/DroidAPITestUtils.java b/droid-api/src/test/java/uk/gov/nationalarchives/droid/internal/api/DroidAPITestUtils.java index 63395760d..18b07ac9f 100644 --- a/droid-api/src/test/java/uk/gov/nationalarchives/droid/internal/api/DroidAPITestUtils.java +++ b/droid-api/src/test/java/uk/gov/nationalarchives/droid/internal/api/DroidAPITestUtils.java @@ -31,12 +31,31 @@ */ package uk.gov.nationalarchives.droid.internal.api; +import com.sun.net.httpserver.HttpServer; +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URLEncodedUtils; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; import jakarta.xml.bind.JAXBException; import org.apache.poi.poifs.filesystem.POIFSFileSystem; import org.w3c.dom.Document; import org.w3c.dom.Element; +import software.amazon.awssdk.services.s3.S3ClientBuilder; import uk.gov.nationalarchives.droid.core.SignatureParseException; - +import java.io.IOException; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.http.HttpClient; +import java.nio.charset.Charset; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Map; +import java.util.stream.Collectors; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; @@ -49,10 +68,6 @@ import java.io.ByteArrayInputStream; import java.io.FileOutputStream; import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Optional; import java.util.zip.GZIPOutputStream; import java.util.zip.ZipOutputStream; @@ -63,13 +78,118 @@ * It makes use of hardcoded signature paths for current version */ public class DroidAPITestUtils { + static Path signaturePath = Paths.get("../droid-results/custom_home/signature_files/DROID_SignatureFile_V119.xml"); static Path containerPath = Paths.get("../droid-results/custom_home/container_sigs/container-signature-20240715.xml"); - public static DroidAPI createApi() throws SignatureParseException { - return DroidAPI.getInstance(signaturePath, containerPath); //Create only once instance of Droid. + public static DroidAPI createApi(URI endpointOverride) throws SignatureParseException { + return createApi(endpointOverride, signaturePath, containerPath); + } + + public static DroidAPI createApi(URI endpointOverride, Path signaturePath, Path containerPath) throws SignatureParseException { + DroidAPI.DroidAPIBuilder droidAPIBuilder = DroidAPI.builder() + .binarySignature(signaturePath) + .containerSignature(containerPath) + .httpClient(HttpClient.newHttpClient()); + S3ClientBuilder builder = S3Client.builder().region(Region.EU_WEST_2); + if(endpointOverride != null) { + S3Client s3Client = builder.endpointOverride(endpointOverride).build(); + return droidAPIBuilder.s3Client(s3Client).build(); + } + return droidAPIBuilder.s3Client(builder.build()).build(); } + static HttpServer createHttpServer() throws IOException { + HttpServer httpServer = HttpServer.create(); + httpServer.createContext("/", exchange -> { + String range = exchange.getRequestHeaders().get("Range").getFirst(); + long size = Files.size(Paths.get(URI.create("file://" + exchange.getRequestURI().toString()))); + byte[] bytesForRange = getBytesForRange(exchange.getRequestURI().getPath(), range); + + exchange.getResponseHeaders().add("Content-Range", range.replace("=", " ") + "/" + size); + exchange.getResponseHeaders().add("Last-Modified", "1970-01-01T00:00:00.000Z"); + exchange.sendResponseHeaders(200, bytesForRange.length); + OutputStream outputStream = exchange.getResponseBody(); + outputStream.write(bytesForRange); + outputStream.close(); + }); + httpServer.bind(new InetSocketAddress(0), 0); + httpServer.start(); + return httpServer; + } + + static HttpServer createS3Server() throws IOException { + HttpServer s3Server = HttpServer.create(); + s3Server.createContext("/", exchange -> { + Map queryParams = URLEncodedUtils + .parse(exchange.getRequestURI(), Charset.defaultCharset()) + .stream().collect(Collectors.toMap(NameValuePair::getName, NameValuePair::getValue)); + if (exchange.getRequestMethod().equals("GET") && queryParams.containsKey("list-type") && queryParams.get("list-type").equals("2")) { + String fileName = queryParams.get("prefix"); + Path filePath = getFilePathFromUriPath("/" + fileName); + long size = Files.size(filePath); + String response = + "" + + "" + + "" + fileName + "" + + "1970-01-01T00:00:00.000Z" + + "" + size + "" + + "" + + ""; + exchange.sendResponseHeaders(200, response.getBytes().length); + OutputStream responseBody = exchange.getResponseBody(); + responseBody.write(response.getBytes()); + responseBody.close(); + } else if (exchange.getRequestMethod().equals("HEAD")) { + String fullPath = exchange.getRequestURI().getPath().substring(1); + Path filePath = getFilePathFromUriPath(fullPath.substring(fullPath.indexOf("/"))); + long size = Files.size(filePath); + exchange.getResponseHeaders().add("Content-Length", Long.toString(size)); + exchange.getResponseHeaders().add("Last-Modified", "Mon, 03 Mar 2025 17:29:48 GMT"); + exchange.sendResponseHeaders(200, -1); + OutputStream responseBody = exchange.getResponseBody(); + responseBody.write("".getBytes()); + responseBody.close(); + } else if (exchange.getRequestMethod().equals("GET")) { + String fullPath = exchange.getRequestURI().getPath().substring(1); + Path filePath = getFilePathFromUriPath(fullPath.substring(fullPath.indexOf("/"))); + String range = exchange.getRequestHeaders().get("Range").getFirst(); + byte[] bytesForRange = getBytesForRange(filePath.toString(), range); + exchange.sendResponseHeaders(200, bytesForRange.length); + OutputStream responseBody = exchange.getResponseBody(); + responseBody.write(bytesForRange); + responseBody.close(); + } + }); + s3Server.bind(new InetSocketAddress(0), 0); + s3Server.start(); + return s3Server; + } + + private static Path getFilePathFromUriPath(String uriPath) { + if(FileSystems.getDefault().getSeparator().equals("\\")) { + return Path.of(uriPath.substring(1) + ); + } else { + return Path.of(uriPath); + } + } + + public static byte[] getBytesForRange(String filePath, String range) { + String[] rangeArr = range.split("=")[1].split("-"); + int rangeStart = Integer.parseInt(rangeArr[0]); + int rangeEnd = Integer.parseInt(rangeArr[1]); + int length = rangeEnd - rangeStart + 1; + try (RandomAccessFile raf = new RandomAccessFile(filePath, "r")) { + raf.seek(rangeStart); + byte[] buffer = new byte[length]; + int bytesRead = raf.read(buffer); + return bytesRead == length ? buffer : Arrays.copyOf(buffer, bytesRead); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + public record ContainerType(String name, String id, String puid) {} public record ContainerFile(ContainerType containerType, String sequence, String puid, Optional path) {} @@ -79,7 +199,7 @@ public static String generateId() { private static Path generateFile(String extension) { try { - return Files.createTempDirectory("test").resolve("test.%sm".formatted(extension)); + return Files.createTempDirectory("test").resolve("test.%s".formatted(extension)); } catch (IOException e) { throw new RuntimeException(e); } @@ -129,11 +249,11 @@ public static Path generateGzFile(String data) { return outputFilePath; } - public static DroidAPI createApiForContainer(ContainerFile signatureFile) { + public static DroidAPI createApiForContainer(URI endpointOverride, ContainerFile signatureFile) { try { Path containerFilePath = generateContainerSignatureFile(signatureFile); Path signatureFilePath = generateSignatureFile(signatureFile.puid, signatureFile.containerType); - return DroidAPI.getInstance(signatureFilePath, containerFilePath); + return createApi(endpointOverride, signatureFilePath, containerFilePath); } catch (ParserConfigurationException | IOException | TransformerException | JAXBException | SignatureParseException e) { throw new RuntimeException(e); diff --git a/droid-build-tools/src/main/resources/checkstyle-main.xml b/droid-build-tools/src/main/resources/checkstyle-main.xml index e1bb8b181..bad8b1d88 100644 --- a/droid-build-tools/src/main/resources/checkstyle-main.xml +++ b/droid-build-tools/src/main/resources/checkstyle-main.xml @@ -101,7 +101,7 @@ - + @@ -155,10 +155,10 @@ - + - + @@ -176,7 +176,7 @@ - + @@ -188,10 +188,6 @@ - - - - diff --git a/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandFactory.java b/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandFactory.java index 992835cda..515077720 100644 --- a/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandFactory.java +++ b/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandFactory.java @@ -96,6 +96,9 @@ public interface CommandFactory { */ DroidCommand getNoProfileCommand(CommandLine cli) throws CommandLineSyntaxException; + DroidCommand getS3Command(CommandLine cli) throws CommandLineSyntaxException; + + DroidCommand getHttpCommand(CommandLine cli) throws CommandLineSyntaxException; /** * @return a new check signature update command. diff --git a/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandFactoryImpl.java b/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandFactoryImpl.java index f83981495..0250d7a2d 100644 --- a/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandFactoryImpl.java +++ b/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandFactoryImpl.java @@ -31,10 +31,8 @@ */ package uk.gov.nationalarchives.droid.command.action; import java.io.PrintWriter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Locale; +import java.net.URI; +import java.util.*; import org.apache.commons.cli.CommandLine; import org.apache.commons.configuration.CombinedConfiguration; @@ -248,6 +246,20 @@ public DroidCommand getProfileCommand(final CommandLine cli) throws CommandLineS @Override public DroidCommand getNoProfileCommand(final CommandLine cli) throws CommandLineSyntaxException { + return getCommand(cli, getNoProfileResources(cli)); + } + + @Override + public DroidCommand getS3Command(CommandLine cli) throws CommandLineSyntaxException { + return getCommand(cli, getS3Resources(cli)); + } + + @Override + public DroidCommand getHttpCommand(CommandLine cli) throws CommandLineSyntaxException { + return getCommand(cli, getHttpResources(cli)); + } + + private DroidCommand getCommand(CommandLine cli, String[] resources) throws CommandLineSyntaxException { final ProfileRunCommand command = context.getProfileRunCommand(); PropertiesConfiguration overrides = getOverrideProperties(cli); @@ -258,7 +270,14 @@ public DroidCommand getNoProfileCommand(final CommandLine cli) throws CommandLin overrides.setProperty(DroidGlobalProperty.QUOTE_ALL_FIELDS.getName(), false); overrides.setProperty(DroidGlobalProperty.COLUMNS_TO_WRITE.getName(), "FILE_PATH PUID"); - command.setResources(getNoProfileResources(cli)); + + if (cli.hasOption(CommandLineParam.HTTP_PROXY.toString())) { + URI proxyUri = URI.create(cli.getOptionValue(CommandLineParam.HTTP_PROXY.toString())); + overrides.setProperty(DroidGlobalProperty.UPDATE_USE_PROXY.getName(), true); + overrides.setProperty(DroidGlobalProperty.UPDATE_PROXY_HOST.getName(), proxyUri.getHost()); + overrides.setProperty(DroidGlobalProperty.UPDATE_PROXY_PORT.getName(), proxyUri.getPort()); + } + command.setResources(resources); command.setDestination(getDestination(cli, overrides)); // will also set the output csv file in overrides if present. command.setRecursive(cli.hasOption(CommandLineParam.RECURSIVE.toString())); command.setProperties(overrides); // must be called after we set destination. @@ -337,6 +356,28 @@ private String[] getNoProfileResources(CommandLine cli) throws CommandLineSyntax return resources; } + private String[] getS3Resources(CommandLine cli) throws CommandLineSyntaxException { + String[] resources = cli.getOptionValues(CommandLineParam.RUN_S3.toString()); + if (resources == null || resources.length == 0) { + resources = cli.getArgs(); // if no profile resources specified, use unbound arguments: + if (resources == null || resources.length == 0) { + throw new CommandLineSyntaxException(NO_RESOURCES_SPECIFIED); + } + } + return resources; + } + + private String[] getHttpResources(CommandLine cli) throws CommandLineSyntaxException { + String[] resources = cli.getOptionValues(CommandLineParam.RUN_HTTP.toString()); + if (resources == null || resources.length == 0) { + resources = cli.getArgs(); // if no profile resources specified, use unbound arguments: + if (resources == null || resources.length == 0) { + throw new CommandLineSyntaxException(NO_RESOURCES_SPECIFIED); + } + } + return resources; + } + private PropertiesConfiguration getOverrideProperties(CommandLine cli) throws CommandLineSyntaxException { PropertiesConfiguration overrideProperties = null; // Get properties from a file: diff --git a/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandLineParam.java b/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandLineParam.java index c849a5846..f61e0aad6 100644 --- a/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandLineParam.java +++ b/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/CommandLineParam.java @@ -294,6 +294,31 @@ public DroidCommand getCommand(CommandFactory commandFactory, CommandLine cli) { } }, + /** Runs without a profile and with the specified S3 object. */ + RUN_S3("S3", "S3-resource", true, -1, I18N.RUN_NO_PROFILE_HELP, "s3Url") { + @Override + public DroidCommand getCommand(CommandFactory commandFactory, CommandLine cli) + throws CommandLineSyntaxException { + return commandFactory.getS3Command(cli); + } + }, + + /** Sets a proxy for use with the HTTP and S3 options. */ + HTTP_PROXY("proxy", "http-proxy", true, -1, I18N.PROXY_HELP, "proxyUrl") { + @Override public DroidCommand getCommand(CommandFactory commandFactory, CommandLine cli) { + return null; + } + }, + + /** Runs without a profile and with the specified http(s) url. */ + RUN_HTTP("HTTP", "HTTP-resource", true, -1, I18N.RUN_NO_PROFILE_HELP, "httpUrl") { + @Override + public DroidCommand getCommand(CommandFactory commandFactory, CommandLine cli) + throws CommandLineSyntaxException { + return commandFactory.getHttpCommand(cli); + } + }, + /** Container signature file. */ CONTAINER_SIGNATURE_FILE("Nc", "container-file", true, 1, I18N.CONTAINER_SIGNATURE_FILE_HELP, filename()) { @@ -412,6 +437,8 @@ public DroidCommand getCommand(CommandFactory commandFactory, CommandLine cli) addTopLevelCommand(REPORT); addTopLevelCommand(LIST_FILTER_FIELD); addTopLevelCommand(RUN_PROFILE); + addTopLevelCommand(RUN_S3); + addTopLevelCommand(RUN_HTTP); addTopLevelCommand(RUN_NO_PROFILE); addTopLevelCommand(CHECK_SIGNATURE_UPDATE); addTopLevelCommand(DOWNLOAD_SIGNATURE_UPDATE); @@ -511,6 +538,19 @@ public static Options options() { topGroup.addOption(param.newOption()); } + addOptions(options); + + options.addOptionGroup(getFilterOptionGroup()); + options.addOptionGroup(getFileFilterOptionGroup()); + options.addOptionGroup(getExportOptionGroup()); + options.addOptionGroup(getExportOutputOptionsGroup()); + options.addOptionGroup(topGroup); + + return options; + + } + + private static void addOptions(Options options) { options.addOption(PROFILES.newOption()); options.addOption(OUTPUT_FILE.newOption()); options.addOption(PROFILE_PROPERTY.newOption()); @@ -530,17 +570,9 @@ public static Options options() { options.addOption(COLUMNS_TO_WRITE.newOption()); options.addOption(QUOTE_COMMAS.newOption()); options.addOption(ROW_PER_FORMAT.newOption()); + options.addOption(HTTP_PROXY.newOption()); options.addOption(JSON_OUTPUT.newOption()); options.addOption(CSV_OUTPUT.newOption()); - - options.addOptionGroup(getFilterOptionGroup()); - options.addOptionGroup(getFileFilterOptionGroup()); - options.addOptionGroup(getExportOptionGroup()); - options.addOptionGroup(getExportOutputOptionsGroup()); - options.addOptionGroup(topGroup); - - return options; - } private static OptionGroup getFileFilterOptionGroup() { diff --git a/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/ProfileRunCommand.java b/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/ProfileRunCommand.java index bbf66ae83..37e215d93 100644 --- a/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/ProfileRunCommand.java +++ b/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/action/ProfileRunCommand.java @@ -44,11 +44,7 @@ import uk.gov.nationalarchives.droid.core.interfaces.signature.SignatureFileInfo; import uk.gov.nationalarchives.droid.core.interfaces.signature.SignatureManager; import uk.gov.nationalarchives.droid.core.interfaces.signature.SignatureType; -import uk.gov.nationalarchives.droid.profile.ProfileInstance; -import uk.gov.nationalarchives.droid.profile.ProfileManager; -import uk.gov.nationalarchives.droid.profile.ProfileResourceFactory; -import uk.gov.nationalarchives.droid.profile.ProfileManagerException; -import uk.gov.nationalarchives.droid.profile.ProfileState; +import uk.gov.nationalarchives.droid.profile.*; import uk.gov.nationalarchives.droid.results.handlers.ProgressObserver; /** @@ -84,7 +80,9 @@ public void execute() throws CommandExecutionException { profile.changeState(ProfileState.VIRGIN); for (String resource : resources) { - profile.addResource(getProfileResourceFactory().getResource(resource, recursive)); + AbstractProfileResource profileResource = getProfileResourceFactory().getResource(resource, recursive); + profileResource.setProxy(profile.getProxy()); + profile.addResource(profileResource); } profileManager.setProgressObserver(profile.getUuid(), null); Future future = profileManager.start(profile.getUuid()); diff --git a/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/i18n/I18N.java b/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/i18n/I18N.java index 915a7d973..e319ffc67 100644 --- a/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/i18n/I18N.java +++ b/droid-command-line/src/main/java/uk/gov/nationalarchives/droid/command/i18n/I18N.java @@ -137,6 +137,9 @@ public final class I18N { /** Run a profile outputting to a csv file or console. */ public static final String RUN_FILE_PROFILE_HELP = "profile.run.file.help"; + /** Configure a proxy to send http requests through for S3 or HTTP identification. */ + public static final String PROXY_HELP = "proxy.help"; + /** Help for signature file. */ public static final String SIGNATURE_FILE_HELP = "signature_file.help"; diff --git a/droid-command-line/src/main/resources/options.properties b/droid-command-line/src/main/resources/options.properties index 5788e1bf9..03e0cf0c0 100644 --- a/droid-command-line/src/main/resources/options.properties +++ b/droid-command-line/src/main/resources/options.properties @@ -98,6 +98,7 @@ profile.rowsPerFormat.help=Outputs a row per format for CSV, rather than a row p profile.json.help=Outputs the results as JSON profile.csv.help=Outputs the results as CSV profile.run.file.help=Adds resources to a new profile which is outputted to a CSV file (or console). Resources are the file path of any file or folder you want to profile. The file paths should be given surrounded in double quotes, and separated by spaces from each other. The profile results will be saved to a single file specified using the -p option. \n For example: droid -Na "C:\\Files\\A Folder" "C:\\Files\\file.xxx" \n Note: You cannot use reporting, filtering and exporting when using the -Na option. +proxy.help=Configure a proxy to send http requests through for S3 or HTTP identification no_profile.run.help=Identify either a specific file, or all files in a folder, without the use of a profile. The file or folder path should be bounded by double quotes. The scan results will be sent to standard output. \n For example: droid -Nr "C:\\Files\\A Folder" \n Note: You cannot use reporting, filtering and exporting when using the -Nr option. signature_file.help=Specify the signature file to be used for identification. Optional if signature file included in path used for -Nr option. container_signature_file.help=[optional] The container signature file to be used for identification. If omitted, container-format files may be identified \ diff --git a/droid-command-line/src/test/java/uk/gov/nationalarchives/droid/command/action/CommandFactoryTest.java b/droid-command-line/src/test/java/uk/gov/nationalarchives/droid/command/action/CommandFactoryTest.java index acf6e2391..5b8a46d52 100644 --- a/droid-command-line/src/test/java/uk/gov/nationalarchives/droid/command/action/CommandFactoryTest.java +++ b/droid-command-line/src/test/java/uk/gov/nationalarchives/droid/command/action/CommandFactoryTest.java @@ -1417,6 +1417,50 @@ public void should_not_expand_any_web_archives_when_hyphen_Wt_flag_is_used_witho assertFalse((boolean)e1.getProperties().getProperty("profile.processWarc")); } + @Test + public void testS3Mode() throws Exception { + when(context.getProfileRunCommand()).thenReturn(profileRunCommand); + String[] args = new String[] { + "-S3", + "s3://bucket/test.doc", + "-A" + }; + CommandLine cli = parse(args); + ProfileRunCommand e1 = (ProfileRunCommand) factory.getS3Command(cli); + assertEquals(e1.getResources()[0], "s3://bucket/test.doc"); + } + + @Test + public void testHttpMode() throws Exception { + when(context.getProfileRunCommand()).thenReturn(profileRunCommand); + String[] args = new String[] { + "-HTTP", + "https://example.com/test.doc", + "-A" + }; + CommandLine cli = parse(args); + ProfileRunCommand e1 = (ProfileRunCommand) factory.getHttpCommand(cli); + assertEquals(e1.getResources()[0], "https://example.com/test.doc"); + } + + @Test + public void testProxyOverride() throws Exception { + when(context.getProfileRunCommand()).thenReturn(profileRunCommand); + String[] args = new String[] { + "-HTTP", + "https://example.com/test.doc", + "-proxy", + "http://localhost:8080", + "-A" + }; + CommandLine cli = parse(args); + ProfileRunCommand e1 = (ProfileRunCommand) factory.getHttpCommand(cli); + assertEquals(e1.getResources()[0], "https://example.com/test.doc"); + assertEquals(e1.getProperties().getProperty("update.proxy"), true); + assertEquals(e1.getProperties().getProperty("update.proxy.host"), "localhost"); + assertEquals(e1.getProperties().getProperty("update.proxy.port"), 8080); + } + /** @Test public void testListReports() throws Exception { diff --git a/droid-container/pom.xml b/droid-container/pom.xml index 2eaf73314..a2eceede8 100644 --- a/droid-container/pom.xml +++ b/droid-container/pom.xml @@ -143,8 +143,9 @@ commons-compress - com.github.tomakehurst - wiremock-jre8 + org.wiremock + wiremock + 3.0.3 test diff --git a/droid-core-interfaces/pom.xml b/droid-core-interfaces/pom.xml index 95de236fc..8d0aff3a2 100644 --- a/droid-core-interfaces/pom.xml +++ b/droid-core-interfaces/pom.xml @@ -55,6 +55,26 @@ + + org.apache.maven.plugins + maven-dependency-plugin + + + analyze + + analyze-only + + + true + + software.amazon.awssdk:sso:jar:${aws.version} + software.amazon.awssdk:ssooidc:jar:${aws.version} + org.apache.logging.log4j:log4j-slf4j2-impl:jar:${log4j2.version} + + + + + @@ -171,8 +191,9 @@ byteseek - junit - junit + org.junit.jupiter + junit-jupiter-api + ${junit.version} test @@ -207,5 +228,45 @@ hamcrest test + + software.amazon.awssdk + s3 + ${aws.version} + + + software.amazon.awssdk + sso + ${aws.version} + + + software.amazon.awssdk + ssooidc + 2.30.17 + + + software.amazon.awssdk + apache-client + ${aws.version} + + + software.amazon.awssdk + http-client-spi + ${aws.version} + + + software.amazon.awssdk + aws-core + ${aws.version} + + + software.amazon.awssdk + regions + ${aws.version} + + + software.amazon.awssdk + sdk-core + ${aws.version} + diff --git a/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/config/DroidGlobalProperty.java b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/config/DroidGlobalProperty.java index 6c0f09b54..594f3bd41 100644 --- a/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/config/DroidGlobalProperty.java +++ b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/config/DroidGlobalProperty.java @@ -162,7 +162,10 @@ public enum DroidGlobalProperty { /** Whether the database plays safe (=true), or gains performance * but loses resilience in the face of failures (=false). */ - DATABASE_DURABILITY("database.durability", PropertyType.BOOLEAN, true); + DATABASE_DURABILITY("database.durability", PropertyType.BOOLEAN, true), + + /** Whether to allow loading files from S3. */ + FILES_FROM_S3("profile.s3", PropertyType.BOOLEAN, true); private static Map allValues = new HashMap(); diff --git a/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/http/S3ClientFactory.java b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/http/S3ClientFactory.java new file mode 100644 index 000000000..a6710705b --- /dev/null +++ b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/http/S3ClientFactory.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core.interfaces.http; + +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.apache.ProxyConfiguration; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; +import software.amazon.awssdk.http.apache.ApacheHttpClient; +import uk.gov.nationalarchives.droid.core.interfaces.signature.ProxySettings; +import uk.gov.nationalarchives.droid.core.interfaces.signature.ProxySubscriber; + +import java.net.URI; + +public class S3ClientFactory implements ProxySubscriber { + + private S3Client s3Client; + + private final Region region; + + public S3ClientFactory(ProxySettings proxySettings) { + proxySettings.addProxySubscriber(this); + setS3Client(proxySettings); + this.region = DefaultAwsRegionProviderChain.builder().build().getRegion(); + } + + @Override + public void onProxyChange(ProxySettings changedProxySettings) { + setS3Client(changedProxySettings); + } + + public S3Client getS3Client() { + return s3Client; + } + + private void setS3Client(ProxySettings clientProxySettings) { + S3ClientBuilder builder = S3Client.builder().region(region); + if (clientProxySettings.isEnabled()) { + ProxyConfiguration proxyConfiguration = ProxyConfiguration.builder() + .endpoint(URI.create("http://" + clientProxySettings.getProxyHost() + ":" + clientProxySettings.getProxyPort())) + .build(); + SdkHttpClient httpClient = ApacheHttpClient.builder() + .proxyConfiguration(proxyConfiguration) + .build(); + this.s3Client = builder.httpClient(httpClient).build(); + } else { + this.s3Client = builder.build(); + } + } +} diff --git a/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/HttpIdentificationRequest.java b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/HttpIdentificationRequest.java new file mode 100644 index 000000000..85820728b --- /dev/null +++ b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/HttpIdentificationRequest.java @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core.interfaces.resource; + +import net.byteseek.io.reader.ReaderInputStream; +import net.byteseek.io.reader.WindowReader; +import net.byteseek.io.reader.cache.TopAndTailFixedLengthCache; +import net.byteseek.io.reader.cache.WindowCache; +import uk.gov.nationalarchives.droid.core.interfaces.IdentificationRequest; +import uk.gov.nationalarchives.droid.core.interfaces.RequestIdentifier; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.nio.file.Path; + + +public class HttpIdentificationRequest implements IdentificationRequest { + + private static final int TOP_TAIL_BUFFER_CAPACITY = 30 * 1024 * 1024; + private HttpWindowReader httpReader; + private final RequestIdentifier identifier; + private final RequestMetaData requestMetaData; + private final long size; + private final HttpClient client; + private HttpUtils.HttpMetadata httpMetadata; + + public HttpIdentificationRequest(final RequestMetaData requestMetaData, final RequestIdentifier identifier, HttpClient httpClient) { + this.identifier = identifier; + this.client = httpClient; + this.requestMetaData = requestMetaData; + this.httpMetadata = new HttpUtils(httpClient).getHttpMetadata(identifier.getUri()); + this.httpReader = buildWindowReader(identifier.getUri()); + this.size = httpMetadata.fileSize(); + } + + private HttpWindowReader buildWindowReader(final URI theFile) { + final WindowCache cache = new TopAndTailFixedLengthCache(this.size, TOP_TAIL_BUFFER_CAPACITY); + HttpUtils.HttpMetadata currentMetadata = this.httpMetadata == null ? new HttpUtils(client).getHttpMetadata(theFile) : this.httpMetadata; + return new HttpWindowReader(cache, currentMetadata, this.client); + } + + /** + * {@inheritDoc} + */ + @Override + public final void open(final URI theFile) throws IOException { + this.httpReader = buildWindowReader(theFile); + httpReader.getWindow(0); + } + + /** + * {@inheritDoc} + */ + @Override + public final String getExtension() { + return ResourceUtils.getExtension(requestMetaData.getName()); + } + + /** + * {@inheritDoc} + */ + @Override + public final String getFileName() { + return requestMetaData.getName(); + } + + /** + * {@inheritDoc} + */ + @Override + public final long size() { + return this.size; + } + + /** + * {@inheritDoc} + */ + @Override + public final void close() throws IOException {} + + /** + * {@inheritDoc} + * + * @throws IOException on failure to get InputStream + */ + @Override + public final InputStream getSourceInputStream() throws IOException { + return new ReaderInputStream(httpReader, false); + } + + /** + * {@inheritDoc} + */ + @Override + public final RequestMetaData getRequestMetaData() { + return requestMetaData; + } + + /** + * @return the identifier + */ + public final RequestIdentifier getIdentifier() { + return identifier; + } + + + @Override + public byte getByte(long position) throws IOException { + final int result = httpReader.readByte(position); + if (result < 0) { + throw new IOException("No byte at position " + position); + } + return (byte) result; + } + + @Override + public WindowReader getWindowReader() { + return this.httpReader; + } + + /** + * Return file associate with identification request. + * + * @return File + */ + public Path getFile() { + return null; + } +} diff --git a/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/HttpUtils.java b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/HttpUtils.java new file mode 100644 index 000000000..3b4522288 --- /dev/null +++ b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/HttpUtils.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core.interfaces.resource; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpHeaders; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Instant; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; + +public class HttpUtils { + + private final HttpClient httpClient; + + public HttpUtils(HttpClient httpClient) { + this.httpClient = httpClient; + } + + public record HttpMetadata(Long fileSize, Long lastModified, URI uri) {} + + public HttpMetadata getHttpMetadata(URI uri) { + HttpRequest request = HttpRequest.newBuilder() + .uri(uri) + .header("Range", "bytes=0-1") + .GET() + .build(); + HttpHeaders headers; + try { + headers = httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()) + .headers(); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + Long lastModified = headers.firstValue("last-modified").map(lastModifiedString -> { + try { + ZonedDateTime parsedDate = ZonedDateTime.parse(lastModifiedString, DateTimeFormatter.RFC_1123_DATE_TIME); + return parsedDate.toEpochSecond(); + } catch (DateTimeParseException e) { + return Instant.now().getEpochSecond(); + } + }).orElse(Instant.now().getEpochSecond()); + Long contentLength = Long.parseLong( + headers + .firstValue("content-range") + .map(range -> range.split("/")[1]) + .orElse("0") + ); + return new HttpMetadata(contentLength, lastModified, uri); + } +} diff --git a/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/HttpWindowReader.java b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/HttpWindowReader.java new file mode 100644 index 000000000..836864e6a --- /dev/null +++ b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/HttpWindowReader.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core.interfaces.resource; + +import net.byteseek.io.reader.AbstractReader; +import net.byteseek.io.reader.cache.WindowCache; +import net.byteseek.io.reader.windows.SoftWindow; +import net.byteseek.io.reader.windows.SoftWindowRecovery; +import net.byteseek.io.reader.windows.Window; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; + +public class HttpWindowReader extends AbstractReader implements SoftWindowRecovery { + + private final HttpClient httpClient; + + private final Long length; + + private final URI uri; + + public HttpWindowReader(WindowCache cache, HttpUtils.HttpMetadata httpMetadata, HttpClient httpClient) { + super(cache); + this.uri = httpMetadata.uri(); + this.httpClient = httpClient; + this.length = httpMetadata.fileSize(); + } + + private HttpResponse responseWithRange(long rangeStart, long rangeEnd) { + HttpRequest request = HttpRequest.newBuilder() + .uri(this.uri) + .header("Range", "bytes=" + rangeStart + "-" + (rangeEnd + this.windowSize - 1)) + .GET() + .build(); + + try { + return httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()); + } catch (InterruptedException | IOException e) { + throw new RuntimeException(e); + } + } + + @Override + protected Window createWindow(long windowStart) throws IOException { + if (windowStart >= 0) { + byte[] bytes = responseWithRange(windowStart, (windowStart + this.windowSize -1)).body(); + int totalRead = bytes.length; + if (totalRead > 0) { + return new SoftWindow(bytes, windowStart, totalRead, this); + } + + } + return null; + } + + @Override + public long length() throws IOException { + return this.length; + } + + @Override + public byte[] reloadWindowBytes(Window window) throws IOException { + return new byte[0]; + } +} diff --git a/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3IdentificationRequest.java b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3IdentificationRequest.java new file mode 100644 index 000000000..73585ba98 --- /dev/null +++ b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3IdentificationRequest.java @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core.interfaces.resource; + +import net.byteseek.io.reader.ReaderInputStream; +import net.byteseek.io.reader.WindowReader; +import net.byteseek.io.reader.cache.TopAndTailFixedLengthCache; +import net.byteseek.io.reader.cache.WindowCache; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Uri; +import uk.gov.nationalarchives.droid.core.interfaces.IdentificationRequest; +import uk.gov.nationalarchives.droid.core.interfaces.RequestIdentifier; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; + + +public class S3IdentificationRequest implements IdentificationRequest { + + private static final int TOP_TAIL_BUFFER_CAPACITY = 30 * 1024 * 1024; + private WindowReader s3Reader; + private final RequestIdentifier identifier; + private final RequestMetaData requestMetaData; + + private final S3Client s3client; + private final S3Utils.S3ObjectMetadata s3ObjectMetadata; + + public S3IdentificationRequest(final RequestMetaData requestMetaData, final RequestIdentifier identifier, final S3Client s3Client) { + this.identifier = identifier; + this.s3client = s3Client; + this.requestMetaData = requestMetaData; + S3Utils s3Utils = new S3Utils(s3Client); + + this.s3ObjectMetadata = s3Utils.getS3ObjectMetadata(identifier.getUri()); + this.s3Reader = buildWindowReader(); + + } + + private WindowReader buildWindowReader() { + final WindowCache cache = new TopAndTailFixedLengthCache(this.s3ObjectMetadata.contentLength(), TOP_TAIL_BUFFER_CAPACITY); + return new S3WindowReader(cache, s3ObjectMetadata, s3client); + } + + /** + * {@inheritDoc} + */ + @Override + public final void open(final S3Uri theFile) throws IOException { + this.s3Reader = buildWindowReader(); + s3Reader.getWindow(0); + } + + /** + * {@inheritDoc} + */ + @Override + public final String getExtension() { + return ResourceUtils.getExtension(requestMetaData.getName()); + } + + /** + * {@inheritDoc} + */ + @Override + public final String getFileName() { + return requestMetaData.getName(); + } + + /** + * {@inheritDoc} + */ + @Override + public final long size() { + return this.s3ObjectMetadata.contentLength(); + } + + /** + * {@inheritDoc} + */ + @Override + public final void close() throws IOException {} + + /** + * {@inheritDoc} + * + * @throws IOException on failure to get InputStream + */ + @Override + public final InputStream getSourceInputStream() throws IOException { + return new ReaderInputStream(s3Reader, false); + } + + /** + * {@inheritDoc} + */ + @Override + public final RequestMetaData getRequestMetaData() { + return requestMetaData; + } + + /** + * @return the identifier + */ + public final RequestIdentifier getIdentifier() { + return identifier; + } + + + @Override + public byte getByte(long position) throws IOException { + final int result = s3Reader.readByte(position); + if (result < 0) { + throw new IOException("No byte at position " + position); + } + return (byte) result; + } + + @Override + public WindowReader getWindowReader() { + return this.s3Reader; + } + + /** + * Return file associate with identification request. + * + * @return File + */ + public Path getFile() { + return null; + } +} diff --git a/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3Utils.java b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3Utils.java new file mode 100644 index 000000000..f0046aee9 --- /dev/null +++ b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3Utils.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core.interfaces.resource; + +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Uri; +import software.amazon.awssdk.services.s3.S3Utilities; +import software.amazon.awssdk.services.s3.model.*; +import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; + +import java.net.URI; +import java.util.Optional; + +public class S3Utils { + + private static final String BUCKET_NOT_FOUND = "Bucket not found in uri "; + + private final S3Client s3Client; + private final Region region; + + public S3Utils(S3Client s3Client) { + this.s3Client = s3Client; + this.region = DefaultAwsRegionProviderChain.builder().build().getRegion(); + } + + public S3Utils(S3Client s3Client, Region region) { + this.s3Client = s3Client; + this.region = region; + } + + public record S3ObjectMetadata(String bucket, Optional key, S3Uri uri, Long contentLength, Long lastModified) {} + + public record S3ObjectList(String bucket, Iterable contents) {} + + public S3ObjectMetadata getS3ObjectMetadata(final URI uri) { + + S3Uri s3Uri = S3Utilities.builder().region(region).build().parseUri(uri); + return getS3ObjectMetadata(s3Uri); + } + + public S3ObjectMetadata getS3ObjectMetadata(final S3Uri s3Uri) { + String bucket = s3Uri.bucket().orElseThrow(() -> new RuntimeException(BUCKET_NOT_FOUND + s3Uri)); + Optional key = s3Uri.key(); + long contentLength = 0L; + long lastModified = 0L; + if (key.isPresent()) { + try { + HeadObjectRequest headObjectRequest = HeadObjectRequest.builder().bucket(bucket).key(key.get()).build(); + HeadObjectResponse headObjectResponse = this.s3Client.headObject(headObjectRequest); + contentLength = headObjectResponse.contentLength(); + lastModified = headObjectResponse.lastModified().getEpochSecond(); + } catch (NoSuchKeyException ignored) {} + } + return new S3ObjectMetadata(bucket, key, s3Uri, contentLength, lastModified); + } + + public S3ObjectList listObjects(final URI uri) { + S3Uri s3Uri = S3Utilities.builder().region(region).build().parseUri(uri); + String bucket = s3Uri.bucket().orElseThrow(() -> new RuntimeException(BUCKET_NOT_FOUND + uri)); + Optional prefix = s3Uri.key(); + + ListObjectsV2Request.Builder builder = ListObjectsV2Request.builder().bucket(bucket); + ListObjectsV2Request request; + if (prefix.isPresent()) { + request = builder.prefix(prefix.get()).build(); + } else { + request = builder.build(); + } + + ListObjectsV2Iterable responseIterable = s3Client.listObjectsV2Paginator(request); + return new S3ObjectList(bucket, responseIterable.contents()); + } +} diff --git a/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3WindowReader.java b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3WindowReader.java new file mode 100644 index 000000000..b471edf3d --- /dev/null +++ b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3WindowReader.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core.interfaces.resource; + +import net.byteseek.io.reader.AbstractReader; +import net.byteseek.io.reader.cache.WindowCache; +import net.byteseek.io.reader.windows.SoftWindow; +import net.byteseek.io.reader.windows.SoftWindowRecovery; +import net.byteseek.io.reader.windows.Window; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; + +public class S3WindowReader extends AbstractReader implements SoftWindowRecovery { + + private static final int BUFFER_LENGTH = 8192; + + private final S3Utils.S3ObjectMetadata s3ObjectMetadata; + + private final S3Client s3Client; + + private final Long length; + + public S3WindowReader(WindowCache cache, S3Utils.S3ObjectMetadata s3ObjectMetadata, S3Client s3Client) { + super(cache); + this.s3Client = s3Client; + this.length = s3ObjectMetadata.contentLength(); + this.s3ObjectMetadata = s3ObjectMetadata; + } + + @Override + protected Window createWindow(long windowStart) throws IOException { + if (windowStart >= 0) { + String key = this.s3ObjectMetadata.key().orElseThrow(() -> new RuntimeException(this.s3ObjectMetadata.key() + " not found")); + GetObjectRequest getS3ObjectRequest = GetObjectRequest.builder() + .bucket(this.s3ObjectMetadata.bucket()) + .key(key) + .range("bytes=" + windowStart + "-" + (windowStart + this.windowSize -1)) + .build(); + + + ResponseInputStream response = s3Client.getObject(getS3ObjectRequest); + byte[] bytes = toByteArray(response); + int totalRead = bytes.length; + response.close(); + if (totalRead > 0) { + return new SoftWindow(bytes, windowStart, totalRead, this); + } + } + return null; + } + + private static byte[] toByteArray(ResponseInputStream inputStream) throws IOException { + try (InputStream in = inputStream; ByteArrayOutputStream out = new ByteArrayOutputStream()) { + byte[] buffer = new byte[BUFFER_LENGTH]; + int bytesRead; + while ((bytesRead = in.read(buffer)) != -1) { + out.write(buffer, 0, bytesRead); + } + return out.toByteArray(); + } + } + + @Override + public long length() throws IOException { + return this.length; + } + + @Override + public byte[] reloadWindowBytes(Window window) throws IOException { + return new byte[0]; + } +} diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/IdentificationRequestFilterTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/IdentificationRequestFilterTest.java index 3291a1a73..b42dee2a6 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/IdentificationRequestFilterTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/IdentificationRequestFilterTest.java @@ -33,7 +33,7 @@ import net.byteseek.io.reader.WindowReader; import org.joda.time.LocalDateTime; -import org.junit.Test; +import org.junit.jupiter.api.Test; import uk.gov.nationalarchives.droid.core.interfaces.filter.BasicFilter; import uk.gov.nationalarchives.droid.core.interfaces.filter.BasicFilterCriterion; @@ -49,7 +49,9 @@ import java.util.Arrays; import java.util.Date; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + public class IdentificationRequestFilterTest { diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ArcArchiveHandlerTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ArcArchiveHandlerTest.java index bbf8ea810..0a279ec41 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ArcArchiveHandlerTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ArcArchiveHandlerTest.java @@ -36,12 +36,11 @@ import java.nio.file.Path; import java.nio.file.Paths; -import static org.junit.Assert.assertEquals; - +import org.junit.jupiter.api.Test; import org.jwat.arc.ArcReaderFactory; import org.jwat.common.ByteCountingPushBackInputStream; -import org.junit.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; /** * @author gseaman diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ArchiveFileUtilsTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ArchiveFileUtilsTest.java index 793aab940..482c78860 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ArchiveFileUtilsTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ArchiveFileUtilsTest.java @@ -38,7 +38,7 @@ import de.waldheinz.fs.util.FileDisk; import net.byteseek.io.reader.FileReader; import net.byteseek.io.reader.WindowReader; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.File; import java.io.IOException; @@ -53,8 +53,8 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.springframework.test.util.AssertionErrors.fail; /** diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ArchiveFormatResolverImplTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ArchiveFormatResolverImplTest.java index 614a12c09..d1312c42f 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ArchiveFormatResolverImplTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ArchiveFormatResolverImplTest.java @@ -31,16 +31,14 @@ */ package uk.gov.nationalarchives.droid.core.interfaces.archive; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestExecutionListeners; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; import org.springframework.test.context.support.DirtiesContextTestExecutionListener; import org.springframework.test.context.transaction.TransactionalTestExecutionListener; @@ -55,7 +53,7 @@ @TestExecutionListeners(listeners = {DependencyInjectionTestExecutionListener.class, DirtiesContextTestExecutionListener.class, TransactionalTestExecutionListener.class}) @ContextConfiguration(locations = "classpath*:archive-spring.xml") //BNO Ignored for now as fails when @RunWith commented out but won't compile if included -@Ignore +@Disabled public class ArchiveFormatResolverImplTest { @Autowired diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ArchiveHandlerFactoryTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ArchiveHandlerFactoryTest.java index 7713d3b94..b12776cb6 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ArchiveHandlerFactoryTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ArchiveHandlerFactoryTest.java @@ -34,11 +34,11 @@ import java.util.HashMap; import java.util.Map; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; /** @@ -47,17 +47,17 @@ */ public class ArchiveHandlerFactoryTest { - private ArchiveHandlerFactoryImpl factory; - private ArchiveHandler zipHandler; - private ArchiveHandler tarHandler; - private ArchiveHandler gzHandler; - private ArchiveHandler arcHandler; - private ArchiveHandler bzHandler; + private static ArchiveHandlerFactoryImpl factory; + private static ArchiveHandler zipHandler; + private static ArchiveHandler tarHandler; + private static ArchiveHandler gzHandler; + private static ArchiveHandler arcHandler; + private static ArchiveHandler bzHandler; - private ArchiveHandler sevenZipHandler; + private static ArchiveHandler sevenZipHandler; - @Before - public void setup() { + @BeforeAll + public static void setup() { factory = new ArchiveHandlerFactoryImpl(); zipHandler = mock(ArchiveHandler.class); diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/BZipArchiveHandlerTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/BZipArchiveHandlerTest.java index 424e16798..57d13f79a 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/BZipArchiveHandlerTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/BZipArchiveHandlerTest.java @@ -43,7 +43,7 @@ import static org.mockito.Mockito.when; import org.apache.commons.compress.compressors.bzip2.BZip2Utils; -import org.junit.Test; +import org.junit.jupiter.api.Test; import uk.gov.nationalarchives.droid.core.interfaces.AsynchDroid; import uk.gov.nationalarchives.droid.core.interfaces.IdentificationRequest; diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ByteseekWindowWrapperTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ByteseekWindowWrapperTest.java index ca6a59ac1..02faa00a8 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ByteseekWindowWrapperTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ByteseekWindowWrapperTest.java @@ -33,10 +33,9 @@ import net.byteseek.io.reader.FileReader; import net.byteseek.io.reader.WindowReader; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.nio.ByteBuffer; @@ -44,7 +43,8 @@ import java.nio.file.Path; import java.nio.file.Paths; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class ByteseekWindowWrapperTest { @@ -52,10 +52,7 @@ public class ByteseekWindowWrapperTest { private WindowReader reader; private ByteseekWindowWrapper windowWrapper; - @Rule - public ExpectedException expectedEx = ExpectedException.none(); - - @Before + @BeforeEach public void setup() throws Exception { reader = getFileReader(RESOURCE_NAME); windowWrapper = new ByteseekWindowWrapper(reader); @@ -100,27 +97,29 @@ public void should_return_negative_value_for_bytes_read_if_current_position_is_b @Test public void should_throw_exception_indicating_that_write_method_is_not_implemented() throws IOException { - expectedEx.expect(IOException.class); - expectedEx.expectMessage("This method from the SeekableByteChannel interface is not implemented"); - windowWrapper.write(ByteBuffer.allocate(10)); + IOException ioException = assertThrows(IOException.class, () -> windowWrapper.write(ByteBuffer.allocate(10))); + assertEquals(ioException.getMessage(), "This method from the SeekableByteChannel interface is not implemented"); } @Test public void should_throw_exception_indicating_that_truncate_method_is_not_implemented() throws IOException { - expectedEx.expect(IOException.class); - expectedEx.expectMessage("This method from the SeekableByteChannel interface is not implemented"); - windowWrapper.truncate(2); + IOException ioException = assertThrows(IOException.class, () -> windowWrapper.truncate(2)); + assertEquals(ioException.getMessage(), "This method from the SeekableByteChannel interface is not implemented"); } @Test public void should_throw_exception_when_trying_to_read_after_closing_the_channel() throws IOException { - expectedEx.expect(ClosedChannelException.class); - ByteBuffer buffer = ByteBuffer.allocate(10); - windowWrapper.read(buffer); - windowWrapper.close(); - buffer.clear(); - windowWrapper.read(buffer); + assertThrows( + ClosedChannelException.class, + () -> { + ByteBuffer buffer = ByteBuffer.allocate(10); + windowWrapper.read(buffer); + windowWrapper.close(); + buffer.clear(); + windowWrapper.read(buffer); + } + ); } - private WindowReader getFileReader(String resourceName) throws IOException { + private static WindowReader getFileReader(String resourceName) throws IOException { Path p = Paths.get("./src/test/resources/" + resourceName); return new FileReader(p.toFile(), 127); // use a small window. } diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/FatArchiveHandlerTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/FatArchiveHandlerTest.java index 6a2103469..2d74d3c13 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/FatArchiveHandlerTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/FatArchiveHandlerTest.java @@ -39,9 +39,9 @@ import java.nio.file.Paths; import org.apache.commons.io.FileUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import uk.gov.nationalarchives.droid.core.interfaces.*; @@ -58,10 +58,10 @@ public class FatArchiveHandlerTest { - private Path tmpDir; + private static Path tmpDir; - @Before - public void setup() throws IOException { + @BeforeAll + public static void setup() throws IOException { tmpDir = Files.createTempDirectory("fat-test"); } @@ -111,8 +111,8 @@ public FatFileIdentificationRequest answer(InvocationOnMock invocationOnMock) { } - @After - public void tearDown(){ + @AfterAll + public static void tearDown(){ FileUtils.deleteQuietly(tmpDir.toFile()); } diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/FatReaderTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/FatReaderTest.java index 22bdaf610..cc97af004 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/FatReaderTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/FatReaderTest.java @@ -34,9 +34,7 @@ import de.waldheinz.fs.ReadOnlyException; import net.byteseek.io.reader.FileReader; import net.byteseek.io.reader.WindowReader; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.*; import java.io.EOFException; import java.io.IOException; @@ -45,28 +43,28 @@ import java.nio.file.Path; import java.nio.file.Paths; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.*; public class FatReaderTest { - private static String RESOURCE_NAME = "saved.zip"; + private static final String RESOURCE_NAME = "saved.zip"; private WindowReader reader; private FatReader fat; - @Before + @BeforeEach public void setup() throws Exception { reader = getFileReader(RESOURCE_NAME); fat = new FatReader(reader); } - @After + @AfterEach public void close() throws Exception { fat.close(); reader.close(); } - private WindowReader getFileReader(String resourceName) throws IOException { + private static WindowReader getFileReader(String resourceName) throws IOException { Path p = Paths.get("./src/test/resources/" + resourceName); return new FileReader(p.toFile(), 127); // use a small odd window size so we cross window boundaries. } @@ -88,9 +86,9 @@ public void testRead() throws Exception { testRead(33, 2); } - @Test(expected = EOFException.class) + @Test public void testReadPastEnd() throws Exception { - testRead(943, 100000); + assertThrows(EOFException.class, () -> testRead(943, 100000)); } private void testRead(long position, int bufferSize) throws Exception { @@ -109,32 +107,31 @@ private void testRead(long position, int bufferSize) throws Exception { assertArrayEquals(backing, expected); } - @Test(expected = ReadOnlyException.class) + @Test public void testwrite() throws Exception { ByteBuffer buffer = ByteBuffer.wrap(new byte[1024]); - fat.write(21, buffer); + assertThrows(ReadOnlyException.class, () -> fat.write(21, buffer)); } - @Test(expected = IllegalStateException.class) + @Test public void testwriteAfterClose() throws Exception { fat.close(); ByteBuffer buffer = ByteBuffer.wrap(new byte[1024]); - fat.write(21, buffer); + assertThrows(IllegalStateException.class, () -> fat.write(21, buffer)); } - @Test(expected = IllegalStateException.class) + @Test public void testflush() throws Exception { fat.flush(); // flushing while open does nothing. fat.close(); - fat.flush(); // should throw IllegalStateException. - + assertThrows(IllegalStateException.class, () -> fat.flush()); } - @Test(expected = IllegalStateException.class) + @Test public void testgetSectorSize() throws Exception { assertEquals(512, fat.getSectorSize()); fat.close(); - fat.getSectorSize(); // should throw IllegalStateException after closing. + assertThrows(IllegalStateException.class, () -> fat.getSectorSize()); } @Test diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/GZipArchiveHandlerTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/GZipArchiveHandlerTest.java index 02e9eba25..668465b00 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/GZipArchiveHandlerTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/GZipArchiveHandlerTest.java @@ -42,7 +42,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import org.junit.Test; +import org.junit.jupiter.api.Test; import uk.gov.nationalarchives.droid.core.interfaces.AsynchDroid; import uk.gov.nationalarchives.droid.core.interfaces.IdentificationRequest; diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ISOImageArchiveHandlerTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ISOImageArchiveHandlerTest.java index 283a7707b..b2f0ce742 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ISOImageArchiveHandlerTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ISOImageArchiveHandlerTest.java @@ -34,8 +34,7 @@ import com.github.stephenc.javaisotools.loopfs.iso9660.Iso9660FileEntry; import com.github.stephenc.javaisotools.loopfs.iso9660.Iso9660FileSystem; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Test; import org.mockito.InOrder; import org.mockito.junit.MockitoJUnitRunner; import uk.gov.nationalarchives.droid.core.interfaces.*; @@ -50,14 +49,13 @@ import java.util.ArrayList; import java.util.List; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.*; /** * Created by rhubner on 2/15/17. */ -@RunWith(MockitoJUnitRunner.class) public class ISOImageArchiveHandlerTest { diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/RarArchiveHandlerTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/RarArchiveHandlerTest.java index 1ab3abf54..e3a8b001d 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/RarArchiveHandlerTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/RarArchiveHandlerTest.java @@ -35,7 +35,7 @@ import com.github.junrar.exception.RarException; import com.github.junrar.volume.FileVolumeManager; import com.github.junrar.rarfile.FileHeader; -import org.junit.Test; +import org.junit.jupiter.api.Test; import uk.gov.nationalarchives.droid.core.interfaces.AsynchDroid; import uk.gov.nationalarchives.droid.core.interfaces.IdentificationRequest; import uk.gov.nationalarchives.droid.core.interfaces.IdentificationResult; @@ -52,7 +52,7 @@ import java.nio.file.Paths; import java.util.List; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.mock; diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/RarReaderTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/RarReaderTest.java index cccd1b00c..a56fe7acb 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/RarReaderTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/RarReaderTest.java @@ -36,16 +36,14 @@ import com.github.junrar.io.SeekableReadOnlyByteChannel; import net.byteseek.io.reader.FileReader; import net.byteseek.io.reader.WindowReader; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.*; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.file.Path; import java.nio.file.Paths; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.*; public class RarReaderTest { @@ -56,7 +54,7 @@ public class RarReaderTest { private Volume vol; private SeekableReadOnlyByteChannel access; - @Before + @BeforeEach public void setup() throws Exception { reader = getFileReader(RESOURCE_NAME); rar = new RarReader(reader); @@ -65,13 +63,13 @@ public void setup() throws Exception { access = vol.getChannel(); } - @After + @AfterEach public void close() throws Exception { archive.close(); reader.close(); } - private WindowReader getFileReader(String resourceName) throws IOException { + private static WindowReader getFileReader(String resourceName) throws IOException { Path p = Paths.get("./src/test/resources/" + resourceName); return new FileReader(p.toFile(), 127); // use a small odd window size so we cross window boundaries. } @@ -101,9 +99,9 @@ public void testPosition() throws Exception { assertEquals(257, access.getPosition()); } - @Test(expected = IOException.class) + @Test public void testSetNegativePosition() throws Exception { - access.setPosition(-23); + assertThrows(IOException.class, () -> access.setPosition(-23)); } @Test diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/SevenZArchiveHandlerTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/SevenZArchiveHandlerTest.java index b30a9c840..72490f93a 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/SevenZArchiveHandlerTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/SevenZArchiveHandlerTest.java @@ -34,7 +34,7 @@ import org.apache.ant.compress.util.SevenZStreamFactory; import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ArchiveInputStream; -import org.junit.Test; +import org.junit.jupiter.api.Test; import uk.gov.nationalarchives.droid.core.interfaces.IdentificationRequest; import uk.gov.nationalarchives.droid.core.interfaces.ResourceId; import uk.gov.nationalarchives.droid.core.interfaces.RequestIdentifier; diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/SevenZipReaderTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/SevenZipReaderTest.java index f94f89399..fc71d986f 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/SevenZipReaderTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/SevenZipReaderTest.java @@ -33,9 +33,7 @@ import net.byteseek.io.reader.FileReader; import net.byteseek.io.reader.WindowReader; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.*; import java.io.IOException; import java.io.RandomAccessFile; @@ -44,27 +42,28 @@ import java.nio.file.Path; import java.nio.file.Paths; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.*; + public class SevenZipReaderTest { - private static String RESOURCE_NAME = "saved.7z"; + private static final String RESOURCE_NAME = "saved.7z"; private WindowReader reader; private SevenZipReader zip; - @Before + @BeforeEach public void setup() throws Exception { reader = getFileReader(RESOURCE_NAME); zip = new SevenZipReader(reader); } - @After + @AfterEach public void close() throws Exception { zip.close(); reader.close(); } - private WindowReader getFileReader(String resourceName) throws IOException { + private static WindowReader getFileReader(String resourceName) throws IOException { Path p = Paths.get("./src/test/resources/" + resourceName); return new FileReader(p.toFile(), 127); // use a small odd window size so we cross window boundaries. } @@ -99,10 +98,10 @@ private void testRead(long position, int bufferSize) throws Exception { assertArrayEquals(backing, expected); } - @Test(expected = NonWritableChannelException.class) + @Test public void testWrite() throws Exception { ByteBuffer buf = ByteBuffer.wrap(new byte[1024]); - zip.write(buf); + assertThrows(NonWritableChannelException.class, () -> zip.write(buf)); } @Test @@ -119,9 +118,9 @@ public void testSize() throws Exception { assertEquals(reader.length(), zip.size()); } - @Test(expected = NonWritableChannelException.class) + @Test public void testTruncate() throws Exception { - zip.truncate(128); + assertThrows(NonWritableChannelException.class, () -> zip.truncate(128)); } @Test diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/TarArchiveHandlerTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/TarArchiveHandlerTest.java index 7e5d0f0b3..d774ba446 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/TarArchiveHandlerTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/TarArchiveHandlerTest.java @@ -41,8 +41,8 @@ import java.util.ArrayList; import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; @@ -54,7 +54,7 @@ import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import uk.gov.nationalarchives.droid.core.interfaces.AsynchDroid; diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/WarcArchiveHandlerTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/WarcArchiveHandlerTest.java index 5c9aa40bd..d73321e2e 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/WarcArchiveHandlerTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/WarcArchiveHandlerTest.java @@ -36,12 +36,12 @@ import java.nio.file.Path; import java.nio.file.Paths; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import org.jwat.warc.WarcReaderFactory; import org.jwat.common.ByteCountingPushBackInputStream; -import org.junit.Test; +import org.junit.jupiter.api.Test; /** * @author gseaman diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ZipEntryRequestFactoryTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ZipEntryRequestFactoryTest.java index cbad62da4..92f7a84fe 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ZipEntryRequestFactoryTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/archive/ZipEntryRequestFactoryTest.java @@ -34,16 +34,16 @@ import java.io.InputStream; import java.net.URI; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import uk.gov.nationalarchives.droid.core.interfaces.IdentificationRequest; import uk.gov.nationalarchives.droid.core.interfaces.RequestIdentifier; @@ -55,10 +55,10 @@ */ public class ZipEntryRequestFactoryTest { - private ZipEntryRequestFactory factory; + private static ZipEntryRequestFactory factory; - @Before - public void setup() { + @BeforeAll + public static void setup() { factory = new ZipEntryRequestFactory(); } diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/filter/BasicFilterCriterionTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/filter/BasicFilterCriterionTest.java index 9ad101650..d6e81152e 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/filter/BasicFilterCriterionTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/filter/BasicFilterCriterionTest.java @@ -31,9 +31,9 @@ */ package uk.gov.nationalarchives.droid.core.interfaces.filter; -import org.junit.Test; +import org.junit.jupiter.api.Test; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.assertEquals; public class BasicFilterCriterionTest { diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/filter/BasicFilterTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/filter/BasicFilterTest.java index 5a0ad553a..9a334f4d5 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/filter/BasicFilterTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/filter/BasicFilterTest.java @@ -31,14 +31,15 @@ */ package uk.gov.nationalarchives.droid.core.interfaces.filter; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.*; public class BasicFilterTest { @@ -46,7 +47,7 @@ public class BasicFilterTest { FilterCriterion bmpCriterion = new BasicFilterCriterion(CriterionFieldEnum.FILE_EXTENSION, CriterionOperator.EQ, "bmp"); FilterCriterion sizeCriterion = new BasicFilterCriterion(CriterionFieldEnum.FILE_SIZE, CriterionOperator.LT, 1000L); - @Before + @BeforeEach public void setup() { criteria.add(bmpCriterion); criteria.add(sizeCriterion); diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/filter/RestrictionFactoryTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/filter/RestrictionFactoryTest.java index b5a76e6a5..c22c72cb5 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/filter/RestrictionFactoryTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/filter/RestrictionFactoryTest.java @@ -33,15 +33,14 @@ import java.util.Date; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import org.joda.time.DateMidnight; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import uk.gov.nationalarchives.droid.core.interfaces.filter.expressions.QueryBuilder; @@ -51,14 +50,14 @@ */ public class RestrictionFactoryTest { - private FilterCriterion filterCriterion; - private QueryBuilder queryBuilder; - private Date date; + private static FilterCriterion filterCriterion; + private static QueryBuilder queryBuilder; + private static Date date; - private Date from; - private Date to; + private static Date from; + private static Date to; - @Before + @BeforeEach public void setup() { queryBuilder = QueryBuilder.forAlias("foo"); diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/filter/StringListParserTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/filter/StringListParserTest.java index a4c48de51..f9496e9b9 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/filter/StringListParserTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/filter/StringListParserTest.java @@ -31,11 +31,11 @@ */ package uk.gov.nationalarchives.droid.core.interfaces.filter; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.List; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.*; public class StringListParserTest { diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/filter/expressions/QueryBuilderTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/filter/expressions/QueryBuilderTest.java index 4754205b7..bbd6f13d8 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/filter/expressions/QueryBuilderTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/filter/expressions/QueryBuilderTest.java @@ -33,9 +33,9 @@ import java.util.Date; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.Test; +import org.junit.jupiter.api.Test; /** * @author rflitcroft diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/hash/MD5HashGeneratorTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/hash/MD5HashGeneratorTest.java index 86bbf18a1..90a4fc9c8 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/hash/MD5HashGeneratorTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/hash/MD5HashGeneratorTest.java @@ -34,10 +34,10 @@ import java.io.IOException; import java.io.InputStream; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; /** * @author rflitcroft @@ -45,10 +45,10 @@ */ public class MD5HashGeneratorTest { - private MD5HashGenerator hashGenerator; + private static MD5HashGenerator hashGenerator; - @Before - public void setup() { + @BeforeAll + public static void setup() { hashGenerator = new MD5HashGenerator(); } diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/hash/SHA1HashGeneratorTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/hash/SHA1HashGeneratorTest.java index bfb42c4dd..ec054edf1 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/hash/SHA1HashGeneratorTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/hash/SHA1HashGeneratorTest.java @@ -34,10 +34,10 @@ import java.io.IOException; import java.io.InputStream; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; /** * @author gseaman @@ -45,10 +45,10 @@ */ public class SHA1HashGeneratorTest { - private SHA1HashGenerator hashGenerator; + private static SHA1HashGenerator hashGenerator; - @Before - public void setup() { + @BeforeAll + public static void setup() { hashGenerator = new SHA1HashGenerator(); } diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/hash/SHA256HashGeneratorTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/hash/SHA256HashGeneratorTest.java index 780c1918b..d46d6013e 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/hash/SHA256HashGeneratorTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/hash/SHA256HashGeneratorTest.java @@ -34,10 +34,10 @@ import java.io.IOException; import java.io.InputStream; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; /** * @author gseaman @@ -47,7 +47,7 @@ public class SHA256HashGeneratorTest { private SHA256HashGenerator hashGenerator; - @Before + @BeforeEach public void setup() { hashGenerator = new SHA256HashGenerator(); } diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/hash/SHA512HashGeneratorTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/hash/SHA512HashGeneratorTest.java index 57b122c70..47142026b 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/hash/SHA512HashGeneratorTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/hash/SHA512HashGeneratorTest.java @@ -32,12 +32,12 @@ package uk.gov.nationalarchives.droid.core.interfaces.hash; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.io.InputStream; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; public class SHA512HashGeneratorTest { diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/CachedBinaryTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/CachedBinaryTest.java index a79e35bae..2c2208004 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/CachedBinaryTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/CachedBinaryTest.java @@ -35,9 +35,9 @@ import net.byteseek.io.reader.cache.AllWindowsCache; import net.byteseek.io.reader.cache.TempFileCache; import net.byteseek.io.reader.windows.Window; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; import java.io.ByteArrayInputStream; import java.nio.file.Files; @@ -45,7 +45,8 @@ import java.nio.file.Paths; import java.util.Random; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; /** @@ -59,8 +60,8 @@ public class CachedBinaryTest { //private CachedByteBuffers cache; - @Before - public void setup() { + @BeforeAll + public static void setup() { } /* @Test @@ -90,7 +91,7 @@ public void testGetInputStreamWithNoBackingFileCache() throws Exception { assertEquals(800, count); } */ - @Test(expected = IndexOutOfBoundsException.class) + @Test public void testGetInputStreamWithNoBackingFileCache1() throws Exception { byte[] rawBytes = new byte[800]; @@ -114,11 +115,11 @@ public void testGetInputStreamWithNoBackingFileCache1() throws Exception { for (int count = 0; count < rawBytes.length; count++) { byteIn = window.getByte(count); - assertEquals("Incorrect byte: " + count, rawBytes[count], (byte) byteIn); + assertEquals(rawBytes[count], (byte) byteIn, "Incorrect byte: " + count); } //This should throw the IndexOutOfBoundsException - byteIn = window.getByte(rawBytes.length); + assertThrows(IndexOutOfBoundsException.class, () -> window.getByte(rawBytes.length)); } } /* @@ -152,8 +153,8 @@ public void testGetInputStreamWithBackingFileCache() throws Exception { assertEquals(8500, count); } */ - @Ignore - @Test(expected = IndexOutOfBoundsException.class) + @Disabled + @Test public void testGetInputStreamWithBackingFileCache1() throws Exception { byte[] rawBytes = new byte[8500]; @@ -180,7 +181,7 @@ public void testGetInputStreamWithBackingFileCache1() throws Exception { for (int count = 0; count < rawBytes.length; count++) { byteIn = window.getByte(count); - assertEquals("Incorrect byte: " + count, rawBytes[count], (byte) byteIn); + assertEquals(rawBytes[count], (byte) byteIn, "Incorrect byte: " + count); } //This should throw the IndexOutOfBoundsException diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/FileSystemIdentificationRequestTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/FileSystemIdentificationRequestTest.java index b3c08e5e1..04647464f 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/FileSystemIdentificationRequestTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/FileSystemIdentificationRequestTest.java @@ -38,29 +38,28 @@ import java.nio.file.Paths; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.*; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import uk.gov.nationalarchives.droid.core.interfaces.RequestIdentifier; public class FileSystemIdentificationRequestTest { - private String fileData; + private static String fileData; - private FileSystemIdentificationRequest fileRequest; - private Path file; + private static FileSystemIdentificationRequest fileRequest; + private static Path file; - private RequestMetaData metaData; - private RequestIdentifier identifier; + private static RequestMetaData metaData; + private static RequestIdentifier identifier; - @Before - public void setup() throws IOException, URISyntaxException { + @BeforeAll + public static void setup() throws IOException, URISyntaxException { - file = Paths.get(getClass().getResource("/testXmlFile.xml").toURI()); + file = Paths.get(FileSystemIdentificationRequestTest.class.getResource("/testXmlFile.xml").toURI()); metaData = new RequestMetaData(Files.size(file), Files.getLastModifiedTime(file).toMillis(), "testXmlFile.xml"); identifier = new RequestIdentifier(file.toUri()); fileRequest = new FileSystemIdentificationRequest( @@ -70,8 +69,8 @@ public void setup() throws IOException, URISyntaxException { fileData = new String(Files.readAllBytes(file), UTF_8); } - @After - public void tearDown() throws IOException { + @AfterAll + public static void tearDown() throws IOException { fileRequest.close(); } diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/GZipIdentificationRequestTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/GZipIdentificationRequestTest.java index 267c8b43b..1e9a06922 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/GZipIdentificationRequestTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/GZipIdentificationRequestTest.java @@ -44,15 +44,13 @@ import org.apache.commons.compress.compressors.gzip.GzipUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import uk.gov.nationalarchives.droid.core.interfaces.RequestIdentifier; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.*; /** * @author rflitcroft @@ -63,19 +61,19 @@ public class GZipIdentificationRequestTest { private static String fileData; - private GZipIdentificationRequest gzRequest; - private Path file; + private static GZipIdentificationRequest gzRequest; + private static Path file; - private RequestMetaData metaData; - private RequestIdentifier identifier; + private static RequestMetaData metaData; + private static RequestIdentifier identifier; - @AfterClass + @AfterAll public static void removeTmpDir() { FileUtils.deleteQuietly(tmpDir.toFile()); } - @BeforeClass + @BeforeAll public static void setupTestData() throws IOException, URISyntaxException { tmpDir = Paths.get("tmp"); Files.createDirectories(tmpDir); @@ -94,10 +92,10 @@ public static void setupTestData() throws IOException, URISyntaxException { } } - @Before - public void setup() throws Exception { + @BeforeAll + public static void setup() throws Exception { - file = Paths.get(getClass().getResource("/testXmlFile.xml.gz").toURI()); + file = Paths.get(GZipIdentificationRequestTest.class.getResource("/testXmlFile.xml.gz").toURI()); metaData = new RequestMetaData(null, null, "foo"); identifier = new RequestIdentifier(URI.create(GzipUtils.getUncompressedFilename(file.toUri().toString()))); @@ -108,8 +106,8 @@ public void setup() throws Exception { gzRequest.open(in); } - @After - public void tearDown() throws IOException { + @AfterAll + public static void tearDown() throws IOException { gzRequest.close(); } diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/HttpIdentificationRequestTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/HttpIdentificationRequestTest.java new file mode 100644 index 000000000..a6406ece4 --- /dev/null +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/HttpIdentificationRequestTest.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core.interfaces.resource; + +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import uk.gov.nationalarchives.droid.core.interfaces.RequestIdentifier; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; +import static uk.gov.nationalarchives.droid.core.interfaces.resource.HttpTestUtils.mockHttpClient; + +public class HttpIdentificationRequestTest { + + @Test + public void testHttpIdentificationRequestGetsFirstWindowOnCreation() throws IOException, InterruptedException { + HttpClient mockHttpClient = mockHttpClient(); + HttpIdentificationRequest request = createRequest(mockHttpClient); + + ArgumentCaptor httpRequestCaptor = ArgumentCaptor.forClass(HttpRequest.class); + verify(mockHttpClient, times(1)).send(httpRequestCaptor.capture(), any(HttpResponse.BodyHandler.class)); + + HttpRequest capturedValue = httpRequestCaptor.getValue(); + + Optional potentialRange = capturedValue.headers().firstValue("Range"); + + assertTrue(potentialRange.isPresent()); + assertEquals(potentialRange.get(), "bytes=0-1"); + } + + @Test + public void testHttpIdentificationRequestHasCorrectAttributes() throws IOException, InterruptedException { + HttpClient mockHttpClient = mockHttpClient(); + HttpIdentificationRequest request = createRequest(mockHttpClient); + + request.open(request.getIdentifier().getUri()); + + assertEquals(request.size(), 4); + assertNull(request.getFile()); + assertEquals(request.getFileName(), "file.txt"); + assertEquals(request.getExtension(), "txt"); + } + + @Test + public void testGetByteReturnsExpectedValues() throws IOException { + HttpClient mockHttpClient = mockHttpClient(); + HttpIdentificationRequest request = createRequest(mockHttpClient); + + request.open(request.getIdentifier().getUri()); + + IOException ioException = assertThrows(IOException.class, () -> request.getByte(-1)); + assertEquals(ioException.getMessage(), "No byte at position -1"); + + IOException outsideRangeException = assertThrows(IOException.class, () -> request.getByte(5)); + assertEquals(outsideRangeException.getMessage(), "No byte at position 5"); + + byte[] testBytes = "test".getBytes(); + for (int i = 0; i < testBytes.length; i++) { + assertEquals(testBytes[i], request.getByte(i)); + } + } + + @Test + public void testCallsEndpointOnceForMultipleRequestsForSameRange() throws IOException, InterruptedException { + HttpClient mockHttpClient = mockHttpClient(); + HttpIdentificationRequest request = createRequest(mockHttpClient); + + request.open(request.getIdentifier().getUri()); + request.getByte(0); + request.getByte(0); + + verify(mockHttpClient, times(2)).send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class)); + } + + @Test + public void testErrorIfHttpCallFails() throws IOException, InterruptedException { + HttpClient mockHttpClient = mock(HttpClient.class); + when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))).thenThrow(RuntimeException.class); + assertThrows(RuntimeException.class, () -> createRequest(mockHttpClient)); + } + + private HttpIdentificationRequest createRequest(HttpClient mockHttpClient) { + RequestMetaData requestMetaData = new RequestMetaData(1L, 1L, "file.txt"); + URI uri = URI.create("https://example.com"); + RequestIdentifier requestIdentifier = new RequestIdentifier(uri); + return new HttpIdentificationRequest(requestMetaData, requestIdentifier, mockHttpClient); + } + +} diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/HttpTestUtils.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/HttpTestUtils.java new file mode 100644 index 000000000..aed6009da --- /dev/null +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/HttpTestUtils.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core.interfaces.resource; + +import java.io.IOException; +import java.net.http.HttpClient; +import java.net.http.HttpHeaders; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class HttpTestUtils { + static HttpClient mockHttpClient() { + HttpClient httpClientMock = mock(HttpClient.class); + + byte[] responseBody = "test".getBytes(); + try { + when(httpClientMock.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))).thenAnswer(invocation -> { + HttpRequest argument = invocation.getArgument(0); + HttpResponse httpResponseMock = mock(HttpResponse.class); + String lastModified = DateTimeFormatter.RFC_1123_DATE_TIME.format(ZonedDateTime.ofInstant(Instant.EPOCH, ZoneId.of("UTC"))); + when(httpResponseMock.headers()).thenReturn(HttpHeaders.of( + Map.of("content-range", List.of("bytes 0-0/4"), "last-modified", List.of(lastModified)), (a, b) -> true + )); + String range = argument.headers().firstValue("Range").orElseThrow(() -> new RuntimeException("Missing range")); + int rangeStart = Integer.parseInt(range.split("=")[1].split("-")[0]); + if (rangeStart > responseBody.length) { + when(httpResponseMock.body()).thenThrow(new RuntimeException("Invalid range")); + } else { + when(httpResponseMock.body()).thenReturn(responseBody); + } + return httpResponseMock; + }); + } catch (InterruptedException | IOException e) { + throw new RuntimeException(e); + } + return httpClientMock; + } +} diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/HttpUtilsTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/HttpUtilsTest.java new file mode 100644 index 000000000..dc081e3e9 --- /dev/null +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/HttpUtilsTest.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core.interfaces.resource; + +import org.junit.jupiter.api.Test; + +import java.net.URI; +import java.net.http.HttpClient; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static uk.gov.nationalarchives.droid.core.interfaces.resource.HttpTestUtils.mockHttpClient; + +public class HttpUtilsTest { + + @Test + public void testGetMetadataReturnsExpectedValues() { + HttpClient httpClient = mockHttpClient(); + URI uri = URI.create("https://example.com"); + HttpUtils.HttpMetadata httpMetadata = new HttpUtils(httpClient).getHttpMetadata(uri); + + assertEquals(httpMetadata.fileSize(), 4); + assertEquals(httpMetadata.lastModified(), 0); + assertEquals(uri.toString(), "https://example.com"); + } +} diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/HttpWindowReaderTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/HttpWindowReaderTest.java new file mode 100644 index 000000000..dd0e12af8 --- /dev/null +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/HttpWindowReaderTest.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core.interfaces.resource; + +import net.byteseek.io.reader.cache.WindowCache; +import net.byteseek.io.reader.windows.SoftWindow; +import net.byteseek.io.reader.windows.Window; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static uk.gov.nationalarchives.droid.core.interfaces.resource.HttpTestUtils.mockHttpClient; + +public class HttpWindowReaderTest { + + @Test + public void testWindowReaderReturnsExpectedWindow() throws Exception { + WindowCache windowCache = mock(WindowCache.class); + HttpClient httpClient = mockHttpClient(); + HttpUtils.HttpMetadata httpMetadata = new HttpUtils.HttpMetadata(4L, 0L, URI.create("https://example.com")); + HttpWindowReader httpWindowReader = new HttpWindowReader(windowCache, httpMetadata, httpClient); + + byte[] testResponse = "test".getBytes(); + for (int i = 0; i < testResponse.length; i++) { + Window window = httpWindowReader.createWindow(i); + assertEquals(window.getClass(), SoftWindow.class); + assertEquals(window.getWindowPosition(), i); + assertEquals(window.length(), testResponse.length); + assertEquals(window.getByte(i), testResponse[i]); + } + } + + @Test + public void testWindowReaderReturnsNullIfPositionLessThanZero() throws Exception { + WindowCache windowCache = mock(WindowCache.class); + HttpClient httpClient = mockHttpClient(); + HttpUtils.HttpMetadata httpMetadata = new HttpUtils.HttpMetadata(4L, 0L, URI.create("https://example.com")); + HttpWindowReader httpWindowReader = new HttpWindowReader(windowCache, httpMetadata, httpClient); + + assertNull(httpWindowReader.createWindow(-1)); + } + + @Test + public void testWindowReaderReturnsErrorOnHttpFailure() throws Exception { + WindowCache windowCache = mock(WindowCache.class); + HttpClient httpClient = mock(HttpClient.class); + HttpUtils.HttpMetadata httpMetadata = new HttpUtils.HttpMetadata(4L, 0L, URI.create("https://example.com")); + when(httpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))).thenThrow(new IOException("Error contacting server")); + assertThrows(RuntimeException.class, () -> new HttpWindowReader(windowCache, httpMetadata, httpClient).getWindow(0)); + } +} diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3IdentificationRequestTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3IdentificationRequestTest.java new file mode 100644 index 000000000..fbc511763 --- /dev/null +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3IdentificationRequestTest.java @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core.interfaces.resource; + +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Uri; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import uk.gov.nationalarchives.droid.core.interfaces.RequestIdentifier; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.URI; +import java.time.Instant; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; +import static uk.gov.nationalarchives.droid.core.interfaces.resource.S3TestUtils.mockS3Client; + +public class S3IdentificationRequestTest { + + @Test + public void testS3IdentificationRequestGetsFirstWindowOnCreation() throws IOException { + S3Client mockS3Client = mockS3Client(); + S3IdentificationRequest request = createRequest(mockS3Client); + S3Uri s3Uri = S3Uri.builder().uri(request.getIdentifier().getUri()).build(); + + request.open(s3Uri); + + ArgumentCaptor getObjectRequestCaptor = ArgumentCaptor.forClass(GetObjectRequest.class); + verify(mockS3Client, times(1)).headObject(any(HeadObjectRequest.class)); + verify(mockS3Client, times(1)).getObject(getObjectRequestCaptor.capture()); + + GetObjectRequest getRequestValue = getObjectRequestCaptor.getValue(); + assertEquals(getRequestValue.range(), "bytes=0-4095"); + } + + @Test + public void testS3IdentificationRequestHasCorrectAttributes() throws IOException { + S3Client mockS3Client = mockS3Client(); + S3IdentificationRequest request = createRequest(mockS3Client); + S3Uri s3Uri = S3Uri.builder().uri(request.getIdentifier().getUri()).build(); + + request.open(s3Uri); + + assertEquals(request.size(), 1); + assertNull(request.getFile()); + assertEquals(request.getFileName(), "entry.txt"); + assertEquals(request.getExtension(), "txt"); + } + + @Test + public void testGetByteReturnsExpectedValues() throws IOException { + S3Client mockS3Client = mockS3Client(); + S3IdentificationRequest request = createRequest(mockS3Client); + S3Uri s3Uri = S3Uri.builder().uri(request.getIdentifier().getUri()).build(); + + request.open(s3Uri); + + IOException ioException = assertThrows(IOException.class, () -> request.getByte(-1)); + assertEquals(ioException.getMessage(), "No byte at position -1"); + + IOException outsideRangeException = assertThrows(IOException.class, () -> request.getByte(5)); + assertEquals(outsideRangeException.getMessage(), "No byte at position 5"); + + byte[] testBytes = "test".getBytes(); + for (int i = 0; i < testBytes.length; i++) { + assertEquals(testBytes[i], request.getByte(i)); + } + } + + @Test + public void testCallsS3OnceForMultipleRequestsForSameRange() throws IOException { + S3Client mockS3Client = mockS3Client(); + S3IdentificationRequest request = createRequest(mockS3Client); + S3Uri s3Uri = S3Uri.builder().uri(request.getIdentifier().getUri()).build(); + + request.open(s3Uri); + request.getByte(0); + request.getByte(0); + + verify(mockS3Client, times(1)).getObject(any(GetObjectRequest.class)); + } + + @Test + public void testErrorIfS3GetObjectCallsFail() { + S3Client mockS3Client = mockS3Client(); + when(mockS3Client.getObject(any(GetObjectRequest.class))).thenThrow(SdkException.class); + S3IdentificationRequest request = createRequest(mockS3Client); + S3Uri s3Uri = S3Uri.builder().uri(request.getIdentifier().getUri()).build(); + + assertThrows(SdkException.class, () -> request.open(s3Uri)); + } + + @Test + public void testErrorIfS3HeadObjectCallsFail() { + S3Client mockS3Client = mock(S3Client.class); + when(mockS3Client.headObject(any(HeadObjectRequest.class))).thenThrow(SdkException.class); + assertThrows(SdkException.class, () -> createRequest(mockS3Client)); + } + + private S3IdentificationRequest createRequest(S3Client mockS3Client) { + RequestMetaData requestMetaData = new RequestMetaData(2L, 1L, "entry.txt"); + URI uri = URI.create("s3://bucket/test"); + + RequestIdentifier requestIdentifier = new RequestIdentifier(uri); + return new S3IdentificationRequest(requestMetaData, requestIdentifier, mockS3Client); + } +} diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3TestUtils.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3TestUtils.java new file mode 100644 index 000000000..94a3a1823 --- /dev/null +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3TestUtils.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core.interfaces.resource; + +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; + +import java.io.ByteArrayInputStream; +import java.time.Instant; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class S3TestUtils { + + static S3Client mockS3Client() { + S3Client mockS3Client = mock(S3Client.class); + HeadObjectResponse response = HeadObjectResponse.builder().contentLength(1L).lastModified(Instant.ofEpochSecond(1)).build(); + GetObjectResponse getObjectResponse = GetObjectResponse.builder().build(); + ResponseInputStream responseInputStream = new ResponseInputStream<>(getObjectResponse, new ByteArrayInputStream("test".getBytes())); + when(mockS3Client.headObject(any(HeadObjectRequest.class))).thenReturn(response); + when(mockS3Client.getObject(any(GetObjectRequest.class))).thenAnswer(invocation -> { + GetObjectRequest argument = invocation.getArgument(0, GetObjectRequest.class); + String range = argument.range(); + byte[] outputBytes = "test".getBytes(); + int rangeStart = Integer.parseInt(range.split("=")[1].split("-")[0]); + if (rangeStart > outputBytes.length) { + throw new IllegalArgumentException("Range start larger than file size"); + } + return new ResponseInputStream<>(getObjectResponse, new ByteArrayInputStream(outputBytes)); + }); + return mockS3Client; + } +} diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3UtilsTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3UtilsTest.java new file mode 100644 index 000000000..8442c7baf --- /dev/null +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3UtilsTest.java @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core.interfaces.resource; + +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Uri; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.S3Object; +import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; + +import java.net.URI; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; +import static uk.gov.nationalarchives.droid.core.interfaces.resource.S3TestUtils.mockS3Client; + +public class S3UtilsTest { + + @Test + public void getObjectMetadataReturnsCorrectMetadata() { + S3Client s3Client = mockS3Client(); + S3Utils s3Utils = new S3Utils(s3Client, Region.EU_WEST_2); + URI uri = URI.create("s3://bucket/key"); + S3Utils.S3ObjectMetadata s3ObjectMetadataFromUri = s3Utils.getS3ObjectMetadata(uri); + S3Uri s3Uri = S3Uri.builder().uri(uri).bucket("bucket").key("key").build(); + S3Utils.S3ObjectMetadata s3ObjectMetadataFromS3Uri = s3Utils.getS3ObjectMetadata(s3Uri); + + for (S3Utils.S3ObjectMetadata s3ObjectMetadata: List.of(s3ObjectMetadataFromUri, s3ObjectMetadataFromS3Uri)) { + assertTrue(s3ObjectMetadata.key().isPresent()); + assertEquals(s3ObjectMetadata.key().get(), "key"); + assertEquals(s3ObjectMetadata.bucket(), "bucket"); + assertEquals(s3ObjectMetadata.uri().uri(), uri); + assertEquals(s3ObjectMetadata.contentLength(), 1); + assertEquals(s3ObjectMetadata.lastModified(), 1); + } + } + + @Test + public void listObjectsReturnsAllItemsWhenItemsArePaginated() { + S3Client s3Client = mock(S3Client.class); + + ListObjectsV2Response firstResponse = generateListObjectsResponse(0, true); + ListObjectsV2Response secondResponse = generateListObjectsResponse(1, false); + + ListObjectsV2Iterable responseIterable = new ListObjectsV2Iterable(s3Client, ListObjectsV2Request.builder().build()); + + ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(ListObjectsV2Request.class); + + when(s3Client.listObjectsV2(requestArgumentCaptor.capture())).thenReturn(firstResponse, secondResponse); + when(s3Client.listObjectsV2Paginator(any(ListObjectsV2Request.class))).thenReturn(responseIterable); + + S3Utils.S3ObjectList objectList = new S3Utils(s3Client).listObjects(URI.create("s3://bucket/key")); + + Iterable contents = objectList.contents(); + List contentsList = new ArrayList<>(); + contents.iterator().forEachRemaining(contentsList::add); + + assertEquals(contentsList.size(), 2); + S3Object firstObject = contentsList.getFirst(); + S3Object secondObject = contentsList.getLast(); + + assertEquals(firstObject.key(), "key0"); + assertEquals(firstObject.lastModified().getEpochSecond(), 0); + assertEquals(firstObject.size(), 0); + assertEquals(firstObject.eTag(), "etag0"); + + assertEquals(secondObject.key(), "key1"); + assertEquals(secondObject.lastModified().getEpochSecond(), 1); + assertEquals(secondObject.size(), 1); + assertEquals(secondObject.eTag(), "etag1"); + + List requestValues = requestArgumentCaptor.getAllValues(); + + assertNull(requestValues.getFirst().continuationToken()); + assertEquals(requestValues.getLast().continuationToken(), "token"); + } + + private ListObjectsV2Response generateListObjectsResponse(int suffix, boolean isTruncated) { + S3Object s3Object = S3Object.builder() + .key("key" + suffix) + .lastModified(Instant.ofEpochSecond(suffix)) + .size((long) suffix) + .eTag("etag" + suffix) + .build(); + + ListObjectsV2Response response; + ListObjectsV2Response.Builder builder = ListObjectsV2Response.builder().contents(s3Object).isTruncated(isTruncated); + if (isTruncated) { + response = builder.nextContinuationToken("token").build(); + } else { + response = builder.build(); + } + return response; + } +} diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3WindowReaderTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3WindowReaderTest.java new file mode 100644 index 000000000..8c09c1092 --- /dev/null +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3WindowReaderTest.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core.interfaces.resource; + +import net.byteseek.io.reader.cache.WindowCache; +import net.byteseek.io.reader.windows.SoftWindow; +import net.byteseek.io.reader.windows.Window; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Uri; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; + +import java.net.URI; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static uk.gov.nationalarchives.droid.core.interfaces.resource.S3TestUtils.mockS3Client; + +public class S3WindowReaderTest { + + @Test + public void testWindowReaderReturnsExpectedWindow() throws Exception { + WindowCache windowCache = mock(WindowCache.class); + S3Client s3Client = mockS3Client(); + S3Uri s3Uri = S3Uri.builder().uri(URI.create("s3://bucket/key")).build(); + S3Utils.S3ObjectMetadata s3ObjectMetadata = new S3Utils.S3ObjectMetadata("bucket", Optional.of("key"), s3Uri, 1L, 1L); + S3WindowReader s3WindowReader = new S3WindowReader(windowCache, s3ObjectMetadata, s3Client); + + byte[] testResponse = "test".getBytes(); + for (int i = 0; i < testResponse.length; i++) { + Window window = s3WindowReader.createWindow(i); + assertEquals(window.getClass(), SoftWindow.class); + assertEquals(window.getWindowPosition(), i); + assertEquals(window.length(), testResponse.length); + assertEquals(window.getByte(i), testResponse[i]); + } + } + + @Test + public void testWindowReaderReturnsNullIfPositionLessThanZero() throws Exception { + WindowCache windowCache = mock(WindowCache.class); + S3Client s3Client = mock(S3Client.class); + S3Uri s3Uri = S3Uri.builder().uri(URI.create("s3://bucket/key")).build(); + S3Utils.S3ObjectMetadata s3ObjectMetadata = new S3Utils.S3ObjectMetadata("bucket", Optional.of("key"), s3Uri, 1L, 1L); + S3WindowReader s3WindowReader = new S3WindowReader(windowCache, s3ObjectMetadata, s3Client); + + assertNull(s3WindowReader.createWindow(-1)); + } + + @Test + public void testWindowReaderReturnsErrorOnS3Failure() throws Exception { + WindowCache windowCache = mock(WindowCache.class); + S3Client s3Client = mock(S3Client.class); + when(s3Client.getObject(any(GetObjectRequest.class))).thenThrow(S3Exception.builder().message("Error contacting s3").build()); + S3Uri s3Uri = S3Uri.builder().uri(URI.create("s3://bucket/key")).build(); + S3Utils.S3ObjectMetadata s3ObjectMetadata = new S3Utils.S3ObjectMetadata("bucket", Optional.of("key"), s3Uri, 1L, 1L); + S3WindowReader s3WindowReader = new S3WindowReader(windowCache, s3ObjectMetadata, s3Client); + + assertThrows(S3Exception.class, () -> s3WindowReader.createWindow(0)); + } +} diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/TarEntryIdentificationRequestTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/TarEntryIdentificationRequestTest.java index 6ac6e3868..45edd4b2b 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/TarEntryIdentificationRequestTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/TarEntryIdentificationRequestTest.java @@ -41,45 +41,43 @@ import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.io.FileUtils; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import uk.gov.nationalarchives.droid.core.interfaces.RequestIdentifier; import uk.gov.nationalarchives.droid.core.interfaces.archive.ArchiveFileUtils; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.*; public class TarEntryIdentificationRequestTest { private static Path tmpDir; - private TarEntryIdentificationRequest tarResource; - private URI fileName; - private TarArchiveInputStream in; - private String entryName; - private long size; - private Date modTime; - private RequestMetaData metaData; - private RequestIdentifier identifier; + private static TarEntryIdentificationRequest tarResource; + private static URI fileName; + private static TarArchiveInputStream in; + private static String entryName; + private static long size; + private static Date modTime; + private static RequestMetaData metaData; + private static RequestIdentifier identifier; - @BeforeClass + @BeforeAll public static void createTmpFileDirectory() throws IOException { tmpDir = Paths.get("tmp"); Files.createDirectories(tmpDir); } - @AfterClass + @AfterAll public static void removeTmpDir() { FileUtils.deleteQuietly(tmpDir.toFile()); } - @Before - public void setup() throws Exception { + @BeforeAll + public static void setup() throws Exception { - fileName = getClass().getResource("/saved.tar").toURI(); + fileName = TarEntryIdentificationRequestTest.class.getResource("/saved.tar").toURI(); in = new TarArchiveInputStream(Files.newInputStream(Paths.get(fileName))); TarArchiveEntry entry; @@ -97,8 +95,8 @@ public void setup() throws Exception { tarResource.open(in); } - @After - public void tearDown() throws IOException { + @AfterAll + public static void tearDown() throws IOException { in.close(); tarResource.close(); } diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/ZipEntryIdentificationRequestTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/ZipEntryIdentificationRequestTest.java index 74c689bcb..f9605f467 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/ZipEntryIdentificationRequestTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/ZipEntryIdentificationRequestTest.java @@ -41,44 +41,42 @@ import java.util.zip.ZipFile; import org.apache.commons.io.FileUtils; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import uk.gov.nationalarchives.droid.core.interfaces.RequestIdentifier; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.*; public class ZipEntryIdentificationRequestTest { private static Path tmpDir; - private ZipEntryIdentificationRequest zipResource; - private URI droidZipFileName; - private InputStream in; - private ZipEntry entry; + private static ZipEntryIdentificationRequest zipResource; + private static URI droidZipFileName; + private static InputStream in; + private static ZipEntry entry; - private RequestMetaData metaData; - private RequestIdentifier identifier; + private static RequestMetaData metaData; + private static RequestIdentifier identifier; - @BeforeClass + @BeforeAll public static void createTmpFileDirectory() throws IOException { tmpDir = Paths.get("tmp"); Files.createDirectories(tmpDir); } - @AfterClass + @AfterAll public static void removeTmpDir() { FileUtils.deleteQuietly(tmpDir.toFile()); } - @Before - public void setup() throws Exception { + @BeforeAll + public static void setup() throws Exception { - droidZipFileName = getClass().getResource("/saved.zip").toURI(); + droidZipFileName = ZipEntryIdentificationRequestTest.class.getResource("/saved.zip").toURI(); ZipFile zip = new ZipFile(Paths.get(droidZipFileName).toFile()); entry = zip.getEntry("profile.xml"); in = zip.getInputStream(entry); @@ -99,8 +97,8 @@ public void setup() throws Exception { //assertNotNull(zipResource.getCache().getSourceFile()); } - @After - public void tearDown() throws IOException { + @AfterAll + public static void tearDown() throws IOException { zipResource.close(); in.close(); } diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/util/DroidUrlFormatTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/util/DroidUrlFormatTest.java index 7d40b5526..c395760ed 100644 --- a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/util/DroidUrlFormatTest.java +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/util/DroidUrlFormatTest.java @@ -31,10 +31,10 @@ */ package uk.gov.nationalarchives.droid.core.interfaces.util; -import org.junit.Test; +import org.junit.jupiter.api.Test; import static org.hamcrest.CoreMatchers.*; -import static org.junit.Assert.assertThat; +import static org.hamcrest.MatcherAssert.assertThat; import java.net.URI; import java.net.URISyntaxException; diff --git a/droid-core/pom.xml b/droid-core/pom.xml index f36e8f615..a3b3159cb 100644 --- a/droid-core/pom.xml +++ b/droid-core/pom.xml @@ -81,6 +81,11 @@ commons-lang commons-lang + + commons-io + commons-io + test + junit junit @@ -91,5 +96,41 @@ mockito-core test + + org.junit.jupiter + junit-jupiter-params + ${junit.version} + test + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + + + software.amazon.awssdk + s3 + ${aws.version} + test + + + software.amazon.awssdk + aws-core + ${aws.version} + test + + + software.amazon.awssdk + regions + ${aws.version} + test + + + software.amazon.awssdk + sdk-core + ${aws.version} + test + diff --git a/droid-core/src/test/java/uk/gov/nationalarchives/droid/core/SkeletonSuiteTest.java b/droid-core/src/test/java/uk/gov/nationalarchives/droid/core/SkeletonSuiteTest.java index e716273f5..00617fa21 100644 --- a/droid-core/src/test/java/uk/gov/nationalarchives/droid/core/SkeletonSuiteTest.java +++ b/droid-core/src/test/java/uk/gov/nationalarchives/droid/core/SkeletonSuiteTest.java @@ -36,27 +36,35 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; +import java.util.*; import org.apache.commons.lang.ArrayUtils; -import org.junit.*; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; - +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Uri; +import software.amazon.awssdk.services.s3.S3Utilities; import uk.gov.nationalarchives.droid.core.interfaces.IdentificationRequest; import uk.gov.nationalarchives.droid.core.interfaces.IdentificationResult; import uk.gov.nationalarchives.droid.core.interfaces.IdentificationResultCollection; import uk.gov.nationalarchives.droid.core.interfaces.RequestIdentifier; import uk.gov.nationalarchives.droid.core.interfaces.resource.FileSystemIdentificationRequest; +import uk.gov.nationalarchives.droid.core.interfaces.resource.HttpIdentificationRequest; import uk.gov.nationalarchives.droid.core.interfaces.resource.RequestMetaData; +import uk.gov.nationalarchives.droid.core.interfaces.resource.S3IdentificationRequest; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Stream; + /** * Created by boreilly on 09/11/2016. * @@ -84,16 +92,28 @@ public class SkeletonSuiteTest { private static final Pattern PuidInFilenamePattern = Pattern.compile("^(x-)?fmt-\\d{1,4}"); private static final Pattern PuidPattern = Pattern.compile("^(x-)?fmt/\\d{1,4}"); - private HashMap filesWithPuids; - private HashMap currentMisidentifiedFiles = new HashMap<>(); - private String[] currentKnownUnidentifiedFiles; - private final List allPaths = new ArrayList<>(); + private static HashMap filesWithPuids; + private static HashMap currentMisidentifiedFiles = new HashMap<>(); + private static String[] currentKnownUnidentifiedFiles; + private static final List allPaths = new ArrayList<>(); + private static BinarySignatureIdentifier droid; + + @BeforeAll + public static void initDroid() { + droid = new BinarySignatureIdentifier(); + droid.setSignatureFile(SIGFILE); + + try { + droid.init(); + } catch (SignatureParseException x) { + assertEquals("Can't parse signature file", x.getMessage()); + } + } - @Before - public void setup() throws IOException{ + public static void setup() throws IOException{ //Hashmap to store filenames and the PUID with which DROId is expected to identify the file. - this.filesWithPuids = new HashMap<>(); + filesWithPuids = new HashMap<>(); final Path fmtDirectory = Paths.get(TEST_FILES_DIR + "fmt"); final Path xfmtDirectory = Paths.get(TEST_FILES_DIR + "x-fmt"); @@ -130,8 +150,8 @@ public void setup() throws IOException{ } // If we haven't got a PUID from the filename in the expected format for any file, don't go any further. assertNotEquals(expectedPuid, NO_PUID); - assertTrue(expectedPuid.matches(this.PuidPattern.pattern())); - this.filesWithPuids.put(filename, expectedPuid); + assertTrue(expectedPuid.matches(PuidPattern.pattern())); + filesWithPuids.put(filename, expectedPuid); } } @@ -143,109 +163,128 @@ public void setup() throws IOException{ "One or more skeleton files is listed for both \"no identifications\" and \"other\" identifications!\n"; inBothLists += "Please review your skeleton file test configuration."; for(String s: currentMisidentifiedFiles.keySet()) { - assertTrue(inBothLists, !ArrayUtils.contains(currentKnownUnidentifiedFiles,s)); + Assertions.assertFalse(ArrayUtils.contains(currentKnownUnidentifiedFiles, s), inBothLists); } } - @Test - public void testBinarySkeletonMatch() throws Exception { - - BinarySignatureIdentifier droid = new BinarySignatureIdentifier(); - droid.setSignatureFile(SIGFILE); - + public static Stream> identificationRequests() { + Stream.Builder> builder = Stream.builder(); try { - droid.init(); - } catch (SignatureParseException x) { - assertEquals("Can't parse signature file", x.getMessage()); + setup(); + for (Path skeletonPath: allPaths) { + String filename = skeletonPath.getFileName().toString(); + URI resourceUri = skeletonPath.toUri(); + String escapedPath = skeletonPath.toString() + .replaceAll(" ", "%20") + .replaceAll("\\\\", "/"); + URI httpUri = URI.create("https://test/" + escapedPath); + URI uri = URI.create("s3://test-bucket/" + escapedPath); + S3Uri s3Uri = S3Utilities.builder().region(Region.EU_WEST_2).build().parseUri(uri); + RequestMetaData metaData = new RequestMetaData( + Files.size(skeletonPath), Files.getLastModifiedTime(skeletonPath).toMillis(), filename); + RequestIdentifier fileIdentifier = new RequestIdentifier(resourceUri); + fileIdentifier.setParentId(1L); + RequestIdentifier s3Identifier = new RequestIdentifier(uri); + s3Identifier.setParentId(1L); + RequestIdentifier httpIdentifier = new RequestIdentifier(httpUri); + httpIdentifier.setParentId(1L); + FileSystemIdentificationRequest fileSystemIdentificationRequest = new FileSystemIdentificationRequest(metaData, fileIdentifier); + fileSystemIdentificationRequest.open(skeletonPath); + builder.add(fileSystemIdentificationRequest); + S3IdentificationRequest s3IdentificationRequest = new S3IdentificationRequest(metaData, s3Identifier, new TestS3Client()); + s3IdentificationRequest.open(s3Uri); + builder.add(s3IdentificationRequest); + + HttpIdentificationRequest httpIdentificationRequest = new HttpIdentificationRequest(metaData, httpIdentifier, new TestHttpClient()); + httpIdentificationRequest.open(httpUri); + builder.add(httpIdentificationRequest); + } + } catch (IOException e) { + throw new RuntimeException(e); } + return builder.build(); + } + @Execution(ExecutionMode.CONCURRENT) + @ParameterizedTest + @MethodSource("identificationRequests") + public void testBinarySkeletonMatch(IdentificationRequest request) throws Exception { int errorCount = 0; //Go through all the skeleton files. Check if the PUID that DROId identifies for the file matches the beginning // of the file name. Or if not, that it is expected to return a different PUID, or none at all. - for(final Path skeletonPath : this.allPaths) { - - URI resourceUri = skeletonPath.toUri(); - String filename = skeletonPath.getFileName().toString(); - - RequestMetaData metaData = new RequestMetaData( - Files.size(skeletonPath), Files.getLastModifiedTime(skeletonPath).toMillis(), filename); - RequestIdentifier identifier = new RequestIdentifier(resourceUri); - identifier.setParentId(1L); + String filename = request.getRequestMetaData().getName(); - IdentificationRequest request = new FileSystemIdentificationRequest(metaData, identifier); - request.open(skeletonPath); - IdentificationResultCollection resultsCollection = droid.matchBinarySignatures(request); - List results = resultsCollection.getResults(); - String expectedPuid = filesWithPuids.get(filename); + IdentificationResultCollection resultsCollection = droid.matchBinarySignatures(request); + List results = resultsCollection.getResults(); + String expectedPuid = filesWithPuids.get(filename); - assertNotEquals(expectedPuid, null); + assertNotEquals(expectedPuid, null); - if (!ArrayUtils.contains(this.currentKnownUnidentifiedFiles, filename)) { + if (!ArrayUtils.contains(this.currentKnownUnidentifiedFiles, filename)) { - // Check if we have any results from DROID - we should have as the file is not in the list of known - // unidentified files. - try { - // Catch assertion failure so we can print an error and continue, checking the total - // error count after all files are processed. - assertTrue(results.size() >= 1); - } catch (AssertionError e) { - System.out.println("No results found for file: " + filename + ". Expected: " + expectedPuid); - errorCount++; - continue; - } - - //If we reach here, we have at least one result from DROID for the current file. - //Allow for more than one identification to be returned by DROID. This is fine as long as one of them - //matches the expected PUID. - String[] puidsIdentified = getPuidsFromIdentification(results); + // Check if we have any results from DROID - we should have as the file is not in the list of known + // unidentified files. + try { + // Catch assertion failure so we can print an error and continue, checking the total + // error count after all files are processed. + assertTrue(results.size() >= 1); + } catch (AssertionError e) { + System.out.println("No results found for file: " + filename + ". Expected: " + expectedPuid); + errorCount++; + } - try { - //Catch assertion failure so we can print error and continue, checking the total error count after - // all files are processed. - Assert.assertTrue(ArrayUtils.contains(puidsIdentified,expectedPuid)); - } catch( AssertionError e) { - //Is this a file where we're expecting a different PUID to the one the filename starts with? - if(this.currentMisidentifiedFiles.containsKey(filename)) { - String expectedWrongPuid = currentMisidentifiedFiles.get(filename); - if(ArrayUtils.contains(puidsIdentified, expectedWrongPuid)) { - System.out.println(String.format("INFO: Skeleton file %s identified by expected \"wrong\"" + - " PUID %s instead of %s.", filename, expectedWrongPuid, expectedPuid)); - } else { - System.out.println(printError(filename, expectedWrongPuid, puidsIdentified)); - errorCount++; - } + //If we reach here, we have at least one result from DROID for the current file. + //Allow for more than one identification to be returned by DROID. This is fine as long as one of them + //matches the expected PUID. + String[] puidsIdentified = getPuidsFromIdentification(results); + + try { + //Catch assertion failure so we can print error and continue, checking the total error count after + // all files are processed. + assertTrue(ArrayUtils.contains(puidsIdentified,expectedPuid)); + } catch( AssertionError e) { + //Is this a file where we're expecting a different PUID to the one the filename starts with? + if(this.currentMisidentifiedFiles.containsKey(filename)) { + String expectedWrongPuid = currentMisidentifiedFiles.get(filename); + if(ArrayUtils.contains(puidsIdentified, expectedWrongPuid)) { + System.out.println(String.format("INFO: Skeleton file %s identified by expected \"wrong\"" + + " PUID %s instead of %s.", filename, expectedWrongPuid, expectedPuid)); } else { - // We expected DROID to identify this file with a PUID matching the start of the filename - so - // this is an unexpected result. - System.out.println(printError(filename, expectedPuid, puidsIdentified)); + System.out.println(printError(filename, expectedWrongPuid, puidsIdentified)); errorCount++; } - + } else { + // We expected DROID to identify this file with a PUID matching the start of the filename - so + // this is an unexpected result. + System.out.println(printError(filename, expectedPuid, puidsIdentified)); + errorCount++; } - } else { - //We expect this file not be identified by its name derived PUID, or by any alternative PUID - try { - //Catch assertion failure so we can print error and continue, checking the total error count after - // all files are processed. - assertTrue(results.size() == 0); - } catch ( AssertionError e) { - String[] puidsIdentifiedForFile = getPuidsFromIdentification(results); + } + } else { + //We expect this file not be identified by its name derived PUID, or by any alternative PUID + try { + //Catch assertion failure so we can print error and continue, checking the total error count after + // all files are processed. + assertTrue(results.size() == 0); + } catch ( AssertionError e) { - System.out.println(printError(filename, NO_PUID, puidsIdentifiedForFile)); - errorCount++; - } + String[] puidsIdentifiedForFile = getPuidsFromIdentification(results); + System.out.println(printError(filename, NO_PUID, puidsIdentifiedForFile)); + errorCount++; } + } - assertEquals(String.format("%1$d error(s) occurred in skeleton file identifications, see earlier messages.", - errorCount), 0, errorCount); + assertEquals(0, errorCount, String.format("%1$d error(s) occurred in skeleton file identifications, see earlier messages.", + errorCount)); + } - private void populateNonAndDifferentlyIdentifiedFiles() throws IOException { + private static void populateNonAndDifferentlyIdentifiedFiles() throws IOException { final List unidentifiedFiles = new ArrayList<>(); @@ -264,7 +303,7 @@ private void populateNonAndDifferentlyIdentifiedFiles() throws IOException { } - this.currentKnownUnidentifiedFiles = unidentifiedFiles.toArray(new String[0]); + currentKnownUnidentifiedFiles = unidentifiedFiles.toArray(new String[0]); //Populate files whihc ar expected to be identified by DROId but the PUID identified is not currently //expected to match the beginning of the file name. @@ -279,7 +318,7 @@ private void populateNonAndDifferentlyIdentifiedFiles() throws IOException { if (!strLine.startsWith("//")) { String filename = strLine.split("\\s+")[0]; String puid = strLine.split("\\s+")[1]; - this.currentMisidentifiedFiles.put(filename, puid); + currentMisidentifiedFiles.put(filename, puid); } } } diff --git a/droid-core/src/test/java/uk/gov/nationalarchives/droid/core/TestHttpClient.java b/droid-core/src/test/java/uk/gov/nationalarchives/droid/core/TestHttpClient.java new file mode 100644 index 000000000..2802f31f9 --- /dev/null +++ b/droid-core/src/test/java/uk/gov/nationalarchives/droid/core/TestHttpClient.java @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core; + +import org.apache.commons.io.FileUtils; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLSession; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.net.Authenticator; +import java.net.CookieHandler; +import java.net.ProxySelector; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpHeaders; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import static uk.gov.nationalarchives.droid.core.TestUtils.getBytesForRange; + +public class TestHttpClient extends HttpClient { + + @Override + public Optional cookieHandler() { + return Optional.empty(); + } + + @Override + public Optional connectTimeout() { + return Optional.empty(); + } + + @Override + public Redirect followRedirects() { + return null; + } + + @Override + public Optional proxy() { + return Optional.empty(); + } + + @Override + public SSLContext sslContext() { + return null; + } + + @Override + public SSLParameters sslParameters() { + return null; + } + + @Override + public Optional authenticator() { + return Optional.empty(); + } + + @Override + public Version version() { + return null; + } + + @Override + public Optional executor() { + return Optional.empty(); + } + + @Override + public HttpResponse send(HttpRequest request, HttpResponse.BodyHandler responseBodyHandler) throws IOException, InterruptedException { + return (HttpResponse)request.headers().firstValue("Range").map(rangeResponse -> { + String filePath = request.uri().getPath().substring(1); + try { + ByteArrayInputStream bais = getBytesForRange(filePath, rangeResponse); + return new HttpResponse(){ + + @Override + public int statusCode() { + return 200; + } + + @Override + public HttpRequest request() { + return null; + } + + @Override + public Optional> previousResponse() { + return Optional.empty(); + } + + @Override + public HttpHeaders headers() { + long size = FileUtils.sizeOf(new File(filePath)); + return HttpHeaders.of( + Map.of("content-range", List.of("bytes 0-0/" + size), "last-modified", List.of("1970-01-01T00:00:00.000Z")), (a, b) -> true + ); + } + + @Override + public byte[] body() { + return bais.readAllBytes(); + } + + @Override + public Optional sslSession() { + return Optional.empty(); + } + + @Override + public URI uri() { + return null; + } + + @Override + public Version version() { + return null; + } + }; + } catch (IOException e) { + throw new RuntimeException(e); + } + + }).orElseThrow(() -> new RuntimeException("Could not connect to server")); + } + + @Override + public CompletableFuture> sendAsync(HttpRequest request, HttpResponse.BodyHandler responseBodyHandler) { + return null; + } + + @Override + public CompletableFuture> sendAsync(HttpRequest request, HttpResponse.BodyHandler responseBodyHandler, HttpResponse.PushPromiseHandler pushPromiseHandler) { + return null; + } +} diff --git a/droid-core/src/test/java/uk/gov/nationalarchives/droid/core/TestS3Client.java b/droid-core/src/test/java/uk/gov/nationalarchives/droid/core/TestS3Client.java new file mode 100644 index 000000000..31137b776 --- /dev/null +++ b/droid-core/src/test/java/uk/gov/nationalarchives/droid/core/TestS3Client.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core; + +import org.apache.commons.io.FileUtils; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.*; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.time.Instant; + +import static uk.gov.nationalarchives.droid.core.TestUtils.getBytesForRange; + +public class TestS3Client implements S3Client { + + @Override + public ResponseInputStream getObject(GetObjectRequest getObjectRequest) throws NoSuchKeyException { + try { + ByteArrayInputStream bais = getBytesForRange(getObjectRequest.key(), getObjectRequest.range()); + return new ResponseInputStream<>(GetObjectResponse.builder().build(), bais); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public HeadObjectResponse headObject(HeadObjectRequest headObjectRequest) throws NoSuchKeyException, AwsServiceException, + SdkClientException { + long size = FileUtils.sizeOf(new File(headObjectRequest.key())); + return HeadObjectResponse.builder().contentLength(size).lastModified(Instant.EPOCH).build(); + } + + @Override + public void close() {} + + @Override + public String serviceName() { + return "test-s3"; + } +} diff --git a/droid-core/src/test/java/uk/gov/nationalarchives/droid/core/TestUtils.java b/droid-core/src/test/java/uk/gov/nationalarchives/droid/core/TestUtils.java new file mode 100644 index 000000000..8bc3136e9 --- /dev/null +++ b/droid-core/src/test/java/uk/gov/nationalarchives/droid/core/TestUtils.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.Arrays; + +public class TestUtils { + + static ByteArrayInputStream getBytesForRange(String filePath, String range) throws IOException { + String[] rangeArr = range.split("=")[1].split("-"); + int rangeStart = Integer.parseInt(rangeArr[0]); + int rangeEnd = Integer.parseInt(rangeArr[1]); + int length = rangeEnd - rangeStart + 1; + try (RandomAccessFile raf = new RandomAccessFile(filePath, "r")) { + raf.seek(rangeStart); + byte[] buffer = new byte[length]; + int bytesRead = raf.read(buffer); + byte[] outputBytes = bytesRead == length ? buffer : Arrays.copyOf(buffer, bytesRead); + return new ByteArrayInputStream(outputBytes); + } + } +} diff --git a/droid-core/src/test/java/uk/gov/nationalarchives/droid/core/fragments/LeftFragmentVariableOffsetTest.java b/droid-core/src/test/java/uk/gov/nationalarchives/droid/core/fragments/LeftFragmentVariableOffsetTest.java index 7eb50e77c..d7483869b 100644 --- a/droid-core/src/test/java/uk/gov/nationalarchives/droid/core/fragments/LeftFragmentVariableOffsetTest.java +++ b/droid-core/src/test/java/uk/gov/nationalarchives/droid/core/fragments/LeftFragmentVariableOffsetTest.java @@ -40,9 +40,8 @@ import java.nio.file.Paths; import java.util.Iterator; import java.util.List; -import org.junit.*; -import static org.junit.Assert.*; +import org.junit.jupiter.api.Test; import uk.gov.nationalarchives.droid.core.BinarySignatureIdentifier; import uk.gov.nationalarchives.droid.core.SignatureParseException; import uk.gov.nationalarchives.droid.core.interfaces.IdentificationRequest; @@ -52,6 +51,9 @@ import uk.gov.nationalarchives.droid.core.interfaces.resource.FileSystemIdentificationRequest; import uk.gov.nationalarchives.droid.core.interfaces.resource.RequestMetaData; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + public class LeftFragmentVariableOffsetTest { diff --git a/droid-core/src/test/resources/junit-platform.properties b/droid-core/src/test/resources/junit-platform.properties new file mode 100644 index 000000000..b10b0e321 --- /dev/null +++ b/droid-core/src/test/resources/junit-platform.properties @@ -0,0 +1 @@ +junit.jupiter.execution.parallel.enabled = true \ No newline at end of file diff --git a/droid-help/src/main/resources/Web pages/Command line control.html b/droid-help/src/main/resources/Web pages/Command line control.html index f62f4c571..4e447b254 100644 --- a/droid-help/src/main/resources/Web pages/Command line control.html +++ b/droid-help/src/main/resources/Web pages/Command line control.html @@ -36,6 +36,17 @@ Command line control + @@ -81,7 +92,18 @@