From e5ba1fc20fdbfa9a41b846d62bde0e800f988a93 Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Fri, 28 May 2021 09:30:28 -0500 Subject: [PATCH 001/106] Rename ImageInfo to Information --- .../v1/{ImageInfo.java => Information.java} | 6 +- ...foFactory.java => InformationFactory.java} | 17 +- .../resource/iiif/v1/InformationResource.java | 6 +- .../resource/iiif/v2/ImageResource.java | 6 +- .../v2/{ImageInfo.java => Information.java} | 16 +- ...foFactory.java => InformationFactory.java} | 33 ++-- .../resource/iiif/v2/InformationResource.java | 10 +- .../resource/iiif/v3/ImageResource.java | 10 +- .../v3/{ImageInfo.java => Information.java} | 6 +- ...foFactory.java => InformationFactory.java} | 37 ++-- .../resource/iiif/v3/InformationResource.java | 8 +- ...yTest.java => InformationFactoryTest.java} | 21 ++- .../iiif/v1/InformationResourceTest.java | 12 +- .../iiif/v1/Version1_1ConformanceTest.java | 2 +- ...yTest.java => InformationFactoryTest.java} | 146 ++++++++-------- .../iiif/v2/InformationResourceTest.java | 12 +- .../iiif/v2/Version2_0ConformanceTest.java | 4 +- ...yTest.java => InformationFactoryTest.java} | 162 +++++++++--------- .../iiif/v3/InformationResourceTest.java | 12 +- .../iiif/v3/Version3_0ConformanceTest.java | 2 +- 20 files changed, 270 insertions(+), 258 deletions(-) rename src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/{ImageInfo.java => Information.java} (90%) rename src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/{ImageInfoFactory.java => InformationFactory.java} (88%) rename src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/{ImageInfo.java => Information.java} (69%) rename src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/{ImageInfoFactory.java => InformationFactory.java} (91%) rename src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/{ImageInfo.java => Information.java} (87%) rename src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/{ImageInfoFactory.java => InformationFactory.java} (90%) rename src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/{ImageInfoFactoryTest.java => InformationFactoryTest.java} (92%) rename src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/{ImageInfoFactoryTest.java => InformationFactoryTest.java} (76%) rename src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/{ImageInfoFactoryTest.java => InformationFactoryTest.java} (74%) diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageInfo.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/Information.java similarity index 90% rename from src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageInfo.java rename to src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/Information.java index 3320be6ed..62ada7bf1 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageInfo.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/Information.java @@ -8,8 +8,8 @@ import java.util.List; /** - * Class whose instances are intended to be serialized to JSON for use in IIIF - * Image Information responses. + * Class whose instances are intended to be serialized as JSON for use in + * information responses. * * @see IIIF Image * API 1.1 @@ -19,7 +19,7 @@ @JsonPropertyOrder({ "@context", "@id", "width", "height", "scale_factors", "tile_width", "tile_height", "formats", "qualities", "profile" }) @JsonInclude(JsonInclude.Include.NON_NULL) -class ImageInfo { +final class Information { @JsonProperty("@context") public final String context = "http://library.stanford.edu/iiif/image-api/1.1/context.json"; diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageInfoFactory.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationFactory.java similarity index 88% rename from src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageInfoFactory.java rename to src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationFactory.java index c0ebeba89..9a63063df 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageInfoFactory.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationFactory.java @@ -12,7 +12,10 @@ import java.util.Set; -final class ImageInfoFactory { +/** + * Builds new {@link Information} instances. + */ +final class InformationFactory { private static final int MIN_SIZE = 64; @@ -21,11 +24,11 @@ final class ImageInfoFactory { */ private static final int DEFAULT_MIN_TILE_SIZE = 512; - ImageInfo newImageInfo(final String imageURI, - final Set availableOutputFormats, - final Info info, - final int imageIndex, - ScaleConstraint scaleConstraint) { + Information newImageInfo(final String imageURI, + final Set availableOutputFormats, + final Info info, + final int imageIndex, + ScaleConstraint scaleConstraint) { if (scaleConstraint == null) { scaleConstraint = new ScaleConstraint(1, 1); } @@ -59,7 +62,7 @@ ImageInfo newImageInfo(final String imageURI, // Create an Info instance, which will eventually be serialized // to JSON and sent as the response body. - final ImageInfo imageInfo = new ImageInfo(); + final Information imageInfo = new Information(); imageInfo.id = imageURI; imageInfo.width = virtualSize.intWidth(); imageInfo.height = virtualSize.intHeight(); diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResource.java index 6dfb1efe8..3a957bc37 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResource.java @@ -40,7 +40,7 @@ public Method[] getSupportedMethods() { } /** - * Writes a JSON-serialized {@link ImageInfo} instance to the response. + * Writes a JSON-serialized {@link Information} instance to the response. */ @Override public void doGET() throws Exception { @@ -79,7 +79,7 @@ public void knowAvailableOutputFormats(Set formats) { .build()) { Info info = handler.handle(); - ImageInfo iiifInfo = new ImageInfoFactory().newImageInfo( + Information iiifInfo = new InformationFactory().newImageInfo( getImageURI(), availableOutputFormats, info, @@ -92,7 +92,7 @@ public void knowAvailableOutputFormats(Set formats) { } } - private void addHeaders(ImageInfo info) { + private void addHeaders(Information info) { getResponse().setHeader("Content-Type", getNegotiatedMediaType()); getResponse().setHeader("Link", String.format("<%s>;rel=\"profile\";", info.profile)); diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageResource.java index ab15fa001..1ec02113d 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageResource.java @@ -20,7 +20,7 @@ import java.util.List; /** - * Handles IIIF Image API 2.x image requests. + * Handles image requests. * * @see Image * Request Operations @@ -160,8 +160,8 @@ private void validateSize(Dimension resultingSize, Dimension virtualSize) throws SizeRestrictedException { final var config = Configuration.getInstance(); if (config.getBoolean(Key.IIIF_RESTRICT_TO_SIZES, false)) { - var factory = new ImageInfoFactory(); - factory.getSizes(virtualSize).stream() + new InformationFactory().getSizes(virtualSize) + .stream() .filter(s -> s.width == resultingSize.intWidth() && s.height == resultingSize.intHeight()) .findAny() diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageInfo.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/Information.java similarity index 69% rename from src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageInfo.java rename to src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/Information.java index 280e0890c..56538adbe 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageInfo.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/Information.java @@ -7,18 +7,18 @@ import java.util.List; /** - *

Class whose instances are intended to be serialized to JSON for use in - * IIIF Image Information responses.

+ *

Class whose instances are intended to be serialized as JSON for use in + * information responses.

* - *

Extends Map in order to support arbitrary keys, and LinkedHashMap in - * order to preserve key order.

+ *

Extends {@link LinkedHashMap} in order to support arbitrary keys and + * preserve key order.

* - * @see IIIF Image - * API 2.0: Image Information + * @see IIIF Image + * API 2.1: Image Information * @see jackson-databind * docs */ -class ImageInfo extends LinkedHashMap { +class Information extends LinkedHashMap { @JsonPropertyOrder({ "width", "height" }) public static final class Size { @@ -29,7 +29,7 @@ public static final class Size { public Size() {} public Size(Integer width, Integer height) { - this.width = width; + this.width = width; this.height = height; } } diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageInfoFactory.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationFactory.java similarity index 91% rename from src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageInfoFactory.java rename to src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationFactory.java index 3059b453b..aee1f8cc0 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageInfoFactory.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationFactory.java @@ -24,10 +24,13 @@ import java.util.Map; import java.util.Set; -final class ImageInfoFactory { +/** + * Builds new {@link Information} instances. + */ +final class InformationFactory { private static final Logger LOGGER = - LoggerFactory.getLogger(ImageInfoFactory.class); + LoggerFactory.getLogger(InformationFactory.class); /** * Will be used if {@link Key#IIIF_MIN_SIZE} is not set. @@ -52,7 +55,7 @@ final class ImageInfoFactory { private double maxScale; private int maxPixels, minSize, minTileSize; - ImageInfoFactory() { + InformationFactory() { var config = Configuration.getInstance(); maxPixels = config.getInt(Key.MAX_PIXELS, 0); maxScale = config.getDouble(Key.MAX_SCALE, Double.MAX_VALUE); @@ -70,11 +73,11 @@ final class ImageInfoFactory { * list. * @param scaleConstraint May be {@code null}. */ - ImageInfo newImageInfo(final Set processorOutputFormats, - final String imageURI, - final Info info, - final int infoImageIndex, - ScaleConstraint scaleConstraint) { + Information newImageInfo(final Set processorOutputFormats, + final String imageURI, + final Info info, + final int infoImageIndex, + ScaleConstraint scaleConstraint) { if (scaleConstraint == null) { scaleConstraint = new ScaleConstraint(1, 1); } @@ -89,7 +92,7 @@ ImageInfo newImageInfo(final Set processorOutputFormats, // Create a Map instance, which will eventually be serialized to JSON // and returned in the response body. - final ImageInfo responseInfo = new ImageInfo<>(); + final Information responseInfo = new Information<>(); responseInfo.put("@context", "http://iiif.io/api/image/2/context.json"); responseInfo.put("@id", imageURI); responseInfo.put("protocol", "http://iiif.io/api/image"); @@ -98,7 +101,7 @@ ImageInfo newImageInfo(final Set processorOutputFormats, // sizes -- this will be a 2^n series that will work for both multi- // and monoresolution images. - final List sizes = getSizes(virtualSize); + final List sizes = getSizes(virtualSize); responseInfo.put("sizes", sizes); // The max reduction factor is the maximum number of times the full @@ -115,7 +118,7 @@ ImageInfo newImageInfo(final Set processorOutputFormats, // calculate a tile size close to minTileSize pixels. // Otherwise, use the smallest multiple of the tile size above that // of image resolution 0. - final List tiles = new ArrayList<>(); + final List tiles = new ArrayList<>(); responseInfo.put("tiles", tiles); info.getImages().forEach(image -> @@ -125,7 +128,7 @@ ImageInfo newImageInfo(final Set processorOutputFormats, minTileSize))); for (Dimension uniqueTileSize : uniqueTileSizes) { - final ImageInfo.Tile tile = new ImageInfo.Tile(); + final Information.Tile tile = new Information.Tile(); tile.width = (int) Math.ceil(uniqueTileSize.width()); tile.height = (int) Math.ceil(uniqueTileSize.height()); // Add every scale factor up to 2^RFmax. @@ -202,10 +205,10 @@ ImageInfo newImageInfo(final Set processorOutputFormats, * @param virtualSize Orientation-aware and {@link ScaleConstraint * scale-constrained} full size. */ - List getSizes(Dimension virtualSize) { + List getSizes(Dimension virtualSize) { // This will be a 2^n series that will work for both multi- and // monoresolution images. - final List sizes = new ArrayList<>(); + final List sizes = new ArrayList<>(); // The min reduction factor is the smallest number of reductions that // are required in order to fit within maxPixels. @@ -222,7 +225,7 @@ List getSizes(Dimension virtualSize) { i *= 2) { final int width = (int) Math.round(virtualSize.width() / i); final int height = (int) Math.round(virtualSize.height() / i); - sizes.add(0, new ImageInfo.Size(width, height)); + sizes.add(0, new Information.Size(width, height)); } return sizes; } diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResource.java index 0f0f7a22d..d71e39d74 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResource.java @@ -17,7 +17,7 @@ import org.slf4j.LoggerFactory; /** - * Handles IIIF Image API 2.x information requests. + * Handles information requests. * * @see Information * Requests @@ -41,7 +41,7 @@ public Method[] getSupportedMethods() { } /** - * Writes a JSON-serialized {@link ImageInfo} instance to the response. + * Writes a JSON-serialized {@link Information} instance to the response. */ @Override public void doGET() throws Exception { @@ -119,16 +119,16 @@ private String getNegotiatedMediaType() { private JacksonRepresentation newRepresentation(Info info, Set availableOutputFormats) { - final ImageInfoFactory factory = new ImageInfoFactory(); + final InformationFactory factory = new InformationFactory(); factory.setDelegateProxy(getDelegateProxy()); - final ImageInfo imageInfo = factory.newImageInfo( + final Information iiifInfo = factory.newImageInfo( availableOutputFormats, getImageURI(), info, getPageIndex(), getMetaIdentifier().getScaleConstraint()); - return new JacksonRepresentation(imageInfo); + return new JacksonRepresentation(iiifInfo); } } diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageResource.java index 8a99ee248..07535d21b 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageResource.java @@ -22,7 +22,7 @@ import java.util.List; /** - * Handles IIIF Image API 3.x image requests. + * Handles image requests. * * @see Image * Requests @@ -169,8 +169,8 @@ private static double getMaxScale() { * @param virtualSize Source image size post-rotation and post-scale * constraint. * @param scale May be {@code null}. - * @param isUpscalingAllowed Whether the {@literal size} URI path component - * begins with {@literal ^}. + * @param isUpscalingAllowed Whether the {@code size} URI path component + * begins with {@code ^}. */ private void validateScale(Dimension virtualSize, Scale scale, @@ -202,8 +202,8 @@ private void validateSize(Dimension virtualSize, Dimension resultingSize) throws SizeRestrictedException { final Configuration config = Configuration.getInstance(); if (config.getBoolean(Key.IIIF_RESTRICT_TO_SIZES, false)) { - var factory = new ImageInfoFactory(); - factory.getSizes(virtualSize).stream() + new InformationFactory().getSizes(virtualSize) + .stream() .filter(s -> s.width == resultingSize.intWidth() && s.height == resultingSize.intHeight()) .findAny() diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageInfo.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/Information.java similarity index 87% rename from src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageInfo.java rename to src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/Information.java index 1f08ff38e..67dcaf22f 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageInfo.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/Information.java @@ -7,8 +7,8 @@ import java.util.List; /** - *

Class whose instances are intended to be serialized to JSON for use in - * IIIF Image Information responses.

+ *

Class whose instances are intended to be serialized as JSON for use in + * information responses.

* *

Extends {@link LinkedHashMap} in order to support arbitrary keys and * preserve key order.

@@ -16,7 +16,7 @@ * @see IIIF * Image API 3.0: Technical Properties */ -class ImageInfo extends LinkedHashMap { +class Information extends LinkedHashMap { @JsonPropertyOrder({ "width", "height" }) public static final class Size { diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageInfoFactory.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationFactory.java similarity index 90% rename from src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageInfoFactory.java rename to src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationFactory.java index 800e0d1cd..4c10dfbff 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageInfoFactory.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationFactory.java @@ -22,10 +22,13 @@ import java.util.Set; import java.util.stream.Collectors; -final class ImageInfoFactory { +/** + * Builds new {@link Information} instances. + */ +final class InformationFactory { private static final Logger LOGGER = - LoggerFactory.getLogger(ImageInfoFactory.class); + LoggerFactory.getLogger(InformationFactory.class); private static final String CONTEXT = "http://iiif.io/api/image/3/context.json"; @@ -50,7 +53,7 @@ final class ImageInfoFactory { private long maxPixels; private int minSize, minTileSize; - ImageInfoFactory() { + InformationFactory() { var config = Configuration.getInstance(); maxPixels = config.getInt(Key.MAX_PIXELS, 0); maxScale = config.getDouble(Key.MAX_SCALE, Double.MAX_VALUE); @@ -67,11 +70,11 @@ final class ImageInfoFactory { * list. * @param scaleConstraint May be {@code null}. */ - ImageInfo newImageInfo(final Set processorOutputFormats, - final String imageURI, - final Info info, - final int infoImageIndex, - ScaleConstraint scaleConstraint) { + Information newImageInfo(final Set processorOutputFormats, + final String imageURI, + final Info info, + final int infoImageIndex, + ScaleConstraint scaleConstraint) { if (scaleConstraint == null) { scaleConstraint = new ScaleConstraint(1, 1); } @@ -86,7 +89,7 @@ ImageInfo newImageInfo(final Set processorOutputFormats, // Create an instance, which will later be serialized as JSON and // returned in the response body. - final ImageInfo responseInfo = new ImageInfo<>(); + final Information responseInfo = new Information<>(); responseInfo.put("@context", CONTEXT); responseInfo.put("id", imageURI); responseInfo.put("type", TYPE); @@ -162,10 +165,10 @@ ImageInfo newImageInfo(final Set processorOutputFormats, * @param virtualSize Orientation-aware and {@link ScaleConstraint * scale-constrained} full size. */ - List getSizes(Dimension virtualSize) { + List getSizes(Dimension virtualSize) { // This will be a 2^n series that will work for both multi- and // monoresolution images. - final List sizes = new ArrayList<>(); + final List sizes = new ArrayList<>(); // The min reduction factor is the smallest number of reductions that // are required in order to fit within maxPixels. @@ -182,7 +185,7 @@ List getSizes(Dimension virtualSize) { i *= 2) { final int width = (int) Math.round(virtualSize.width() / i); final int height = (int) Math.round(virtualSize.height() / i); - sizes.add(0, new ImageInfo.Size(width, height)); + sizes.add(0, new Information.Size(width, height)); } return sizes; } @@ -199,10 +202,10 @@ List getSizes(Dimension virtualSize) { * deliver (which may or may not match the physical tile size or a multiple * of it). */ - List getTiles(Dimension virtualSize, - Orientation orientation, - List images) { - final List tiles = new ArrayList<>(); + List getTiles(Dimension virtualSize, + Orientation orientation, + List images) { + final List tiles = new ArrayList<>(); final Set uniqueTileSizes = new HashSet<>(); images.forEach(image -> @@ -217,7 +220,7 @@ List getTiles(Dimension virtualSize, ImageInfoUtil.maxReductionFactor(virtualSize, minSize); for (Dimension uniqueTileSize : uniqueTileSizes) { - final ImageInfo.Tile tile = new ImageInfo.Tile(); + final Information.Tile tile = new Information.Tile(); tile.width = (int) Math.ceil(uniqueTileSize.width()); tile.height = (int) Math.ceil(uniqueTileSize.height()); // Add every scale factor up to 2^RFmax. diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResource.java index 6f9e2306d..7f8fd2b21 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResource.java @@ -41,7 +41,7 @@ public Method[] getSupportedMethods() { } /** - * Writes a JSON-serialized {@link ImageInfo} instance to the response. + * Writes a JSON-serialized {@link Information} instance to the response. */ @Override public void doGET() throws Exception { @@ -121,16 +121,16 @@ private String getNegotiatedContentType() { private JacksonRepresentation newRepresentation(Info info, Set availableOutputFormats) { - final ImageInfoFactory factory = new ImageInfoFactory(); + final InformationFactory factory = new InformationFactory(); factory.setDelegateProxy(getDelegateProxy()); - final ImageInfo imageInfo = factory.newImageInfo( + final Information iiifInfo = factory.newImageInfo( availableOutputFormats, getImageURI(), info, getPageIndex(), getMetaIdentifier().getScaleConstraint()); - return new JacksonRepresentation(imageInfo); + return new JacksonRepresentation(iiifInfo); } } diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageInfoFactoryTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationFactoryTest.java similarity index 92% rename from src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageInfoFactoryTest.java rename to src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationFactoryTest.java index f5db42c76..ff0d5c304 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageInfoFactoryTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationFactoryTest.java @@ -15,10 +15,10 @@ import static org.junit.jupiter.api.Assertions.*; -public class ImageInfoFactoryTest extends BaseTest { +public class InformationFactoryTest extends BaseTest { private String imageUri; - private ImageInfo imageInfo; + private Information imageInfo; private Processor processor; @BeforeEach @@ -35,9 +35,12 @@ public void setUp() throws Exception { TestUtil.getImage("jpg-rgb-594x522x8-baseline.jpg")); Info info = processor.readInfo(); - imageInfo = new ImageInfoFactory().newImageInfo( - imageUri, processor.getAvailableOutputFormats(), - info, 0, new ScaleConstraint(1, 1)); + imageInfo = new InformationFactory().newImageInfo( + imageUri, + processor.getAvailableOutputFormats(), + info, + 0, + new ScaleConstraint(1, 1)); } @Override @@ -56,7 +59,7 @@ private void setUpForRotatedImage() throws Exception { TestUtil.getImage("jpg-xmp-orientation-90.jpg")); Info info = processor.readInfo(); - imageInfo = new ImageInfoFactory().newImageInfo( + imageInfo = new InformationFactory().newImageInfo( imageUri, processor.getAvailableOutputFormats(), info, 0, new ScaleConstraint(1, 1)); } @@ -68,7 +71,7 @@ private void setUpForScaleConstrainedImage() throws Exception { TestUtil.getImage("jpg-rgb-594x522x8-baseline.jpg")); Info info = processor.readInfo(); - imageInfo = new ImageInfoFactory().newImageInfo( + imageInfo = new InformationFactory().newImageInfo( imageUri, processor.getAvailableOutputFormats(), info, 0, new ScaleConstraint(1, 2)); } @@ -160,7 +163,7 @@ void newImageInfoTileWidthWithTiledImage() throws Exception { ((FileProcessor) processor).setSourceFile( TestUtil.getImage("tif-rgb-1res-64x56x8-tiled-uncompressed.tif")); Info info = processor.readInfo(); - imageInfo = new ImageInfoFactory().newImageInfo( + imageInfo = new InformationFactory().newImageInfo( imageUri, processor.getAvailableOutputFormats(), info, 0, new ScaleConstraint(1, 1)); @@ -191,7 +194,7 @@ void newImageInfoTileHeightWithTiledImage() throws Exception { ((FileProcessor) processor).setSourceFile( TestUtil.getImage("tif-rgb-1res-64x56x8-tiled-uncompressed.tif")); Info info = processor.readInfo(); - imageInfo = new ImageInfoFactory().newImageInfo( + imageInfo = new InformationFactory().newImageInfo( imageUri, processor.getAvailableOutputFormats(), info, 0, new ScaleConstraint(1, 1)); diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResourceTest.java index 8e0753b1a..cd9e45876 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResourceTest.java @@ -437,7 +437,7 @@ void testGETURIsInJSON() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://localhost:" + HTTP_PORT + Route.IIIF_1_PATH + "/" + IMAGE, info.id); } @@ -452,7 +452,7 @@ void testGETURIsInJSONWithBaseURIOverride() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://example.org" + Route.IIIF_1_PATH + "/" + IMAGE, info.id); } @@ -468,7 +468,7 @@ void testGETURIsInJSONWithSlashSubstitution() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://localhost:" + HTTP_PORT + Route.IIIF_1_PATH + path, info.id); } @@ -484,7 +484,7 @@ void testGETURIsInJSONWithEncodedCharacters() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://localhost:" + HTTP_PORT + Route.IIIF_1_PATH + path, info.id); } @@ -502,7 +502,7 @@ void testGETURIsInJSONWithProxyHeaders() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://example.org:8080/cats" + Route.IIIF_1_PATH + "/originalID", info.id); } @@ -521,7 +521,7 @@ void testGETBaseURIOverridesProxyHeaders() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("https://example.net" + Route.IIIF_1_PATH + "/" + IMAGE, info.id); } diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/Version1_1ConformanceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/Version1_1ConformanceTest.java index 0c3b089f4..b99b2b38d 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/Version1_1ConformanceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/Version1_1ConformanceTest.java @@ -526,7 +526,7 @@ void testInformationRequestContentType() throws Exception { */ @Test void testInformationRequestJSON() { - // this will be tested in ImageInfoFactoryTest + // this will be tested in InformationFactoryTest } /** diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageInfoFactoryTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationFactoryTest.java similarity index 76% rename from src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageInfoFactoryTest.java rename to src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationFactoryTest.java index 7d311e8ed..2a6907c6a 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageInfoFactoryTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationFactoryTest.java @@ -17,21 +17,21 @@ import static org.junit.jupiter.api.Assertions.*; -public class ImageInfoFactoryTest extends BaseTest { +public class InformationFactoryTest extends BaseTest { private static final Set PROCESSOR_FORMATS = Set.of( Format.get("gif"), Format.get("jpg"), Format.get("png")); - private ImageInfoFactory instance; + private InformationFactory instance; @BeforeEach public void setUp() throws Exception { super.setUp(); - instance = new ImageInfoFactory(); + instance = new InformationFactory(); } - private ImageInfo invokeNewImageInfo() { + private Information invokeNewImageInfo() { final String imageURI = "http://example.org/bla"; final Info info = Info.builder().withSize(1500, 1200).build(); return instance.newImageInfo(PROCESSOR_FORMATS, imageURI, info, 0, @@ -40,26 +40,26 @@ private ImageInfo invokeNewImageInfo() { @Test void testNewImageInfoContext() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals("http://iiif.io/api/image/2/context.json", info.get("@context")); } @Test void testNewImageInfoID() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals("http://example.org/bla", info.get("@id")); } @Test void testNewImageInfoProtocol() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals("http://iiif.io/api/image", info.get("protocol")); } @Test void testNewImageInfoWidth() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals(1500, info.get("width")); } @@ -75,10 +75,10 @@ public Orientation getOrientation() { } }) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); - assertEquals(1200, imageInfo.get("width")); + assertEquals(1200, iiifInfo.get("width")); } @Test @@ -87,15 +87,15 @@ void testNewImageInfoWidthWithScaleConstrainedImage() { final Info info = Info.builder() .withSize(1499, 1199) // test rounding .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 2)); - assertEquals(750, imageInfo.get("width")); + assertEquals(750, iiifInfo.get("width")); } @Test void testNewImageInfoHeight() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals(1200, info.get("height")); } @@ -111,10 +111,10 @@ public Orientation getOrientation() { } }) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); - assertEquals(1500, imageInfo.get("height")); + assertEquals(1500, iiifInfo.get("height")); } @Test @@ -123,19 +123,19 @@ void testNewImageInfoHeightWithScaleConstrainedImage() { final Info info = Info.builder() .withSize(1499, 1199) // test rounding .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 2)); - assertEquals(600, imageInfo.get("height")); + assertEquals(600, iiifInfo.get("height")); } @Test void testNewImageInfoSizes() { - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); @SuppressWarnings("unchecked") - List sizes = - (List) imageInfo.get("sizes"); + List sizes = + (List) iiifInfo.get("sizes"); assertEquals(5, sizes.size()); assertEquals(94, (int) sizes.get(0).width); assertEquals(75, (int) sizes.get(0).height); @@ -152,11 +152,11 @@ void testNewImageInfoSizes() { @Test void testNewImageInfoSizesMinSize() { instance.setMinSize(500); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); @SuppressWarnings("unchecked") - List sizes = - (List) imageInfo.get("sizes"); + List sizes = + (List) iiifInfo.get("sizes"); assertEquals(2, sizes.size()); assertEquals(750, (int) sizes.get(0).width); assertEquals(600, (int) sizes.get(0).height); @@ -167,11 +167,11 @@ void testNewImageInfoSizesMinSize() { @Test void testNewImageInfoSizesMaxSize() { instance.setMaxPixels(10000); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); @SuppressWarnings("unchecked") - List sizes = - (List) imageInfo.get("sizes"); + List sizes = + (List) iiifInfo.get("sizes"); assertEquals(1, sizes.size()); assertEquals(94, (int) sizes.get(0).width); assertEquals(75, (int) sizes.get(0).height); @@ -189,12 +189,12 @@ public Orientation getOrientation() { } }) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); @SuppressWarnings("unchecked") - List sizes = - (List) imageInfo.get("sizes"); + List sizes = + (List) iiifInfo.get("sizes"); assertEquals(5, sizes.size()); assertEquals(75, (int) sizes.get(0).width); assertEquals(94, (int) sizes.get(0).height); @@ -214,12 +214,12 @@ void testNewImageInfoSizesWithScaleConstrainedImage() { final Info info = Info.builder() .withSize(1500, 1200) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 2)); @SuppressWarnings("unchecked") - List sizes = - (List) imageInfo.get("sizes"); + List sizes = + (List) iiifInfo.get("sizes"); assertEquals(4, sizes.size()); assertEquals(94, (int) sizes.get(0).width); assertEquals(75, (int) sizes.get(0).height); @@ -233,11 +233,11 @@ void testNewImageInfoSizesWithScaleConstrainedImage() { @Test void testNewImageInfoTilesWithUntiledMonoResolutionImage() { - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(1, tiles.size()); assertEquals(512, (int) tiles.get(0).width); assertEquals(512, (int) tiles.get(0).height); @@ -257,12 +257,12 @@ void testNewImageInfoTilesWithUntiledMultiResolutionImage() { .withSize(3000, 2000) .withNumResolutions(3) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(1, tiles.size()); assertEquals(512, (int) tiles.get(0).width); assertEquals(512, (int) tiles.get(0).height); @@ -284,12 +284,12 @@ void testNewImageInfoMinTileSize() { .withTileSize(1000, 1000) .build(); instance.setMinTileSize(1000); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(1000, (int) tiles.get(0).width); assertEquals(1000, (int) tiles.get(0).height); } @@ -307,12 +307,12 @@ public Orientation getOrientation() { }) .withTileSize(64, 56) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(56, (int) tiles.get(0).width); assertEquals(64, (int) tiles.get(0).height); } @@ -324,12 +324,12 @@ void testNewImageInfoTilesWithScaleConstrainedImage() { .withSize(64, 56) .withTileSize(64, 56) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 2)); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(32, (int) tiles.get(0).width); assertEquals(28, (int) tiles.get(0).height); } @@ -341,12 +341,12 @@ void testNewImageInfoTilesWithTiledImage() { .withSize(64, 56) .withTileSize(64, 56) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(1, tiles.size()); assertEquals(64, (int) tiles.get(0).width); assertEquals(56, (int) tiles.get(0).height); @@ -357,15 +357,15 @@ void testNewImageInfoTilesWithTiledImage() { @Test void testNewImageInfoProfile() { - ImageInfo imageInfo = invokeNewImageInfo(); - List profile = (List) imageInfo.get("profile"); + Information iiifInfo = invokeNewImageInfo(); + List profile = (List) iiifInfo.get("profile"); assertEquals("http://iiif.io/api/image/2/level2.json", profile.get(0)); } @Test void testNewImageInfoFormats() { - ImageInfo imageInfo = invokeNewImageInfo(); - List profile = (List) imageInfo.get("profile"); + Information iiifInfo = invokeNewImageInfo(); + List profile = (List) iiifInfo.get("profile"); // If some are present, we will assume the rest are. (The exact // contents of the sets are processor-dependent.) assertTrue(((Set) ((Map) profile.get(1)).get("formats")).contains("gif")); @@ -373,8 +373,8 @@ void testNewImageInfoFormats() { @Test void testNewImageInfoQualities() { - ImageInfo imageInfo = invokeNewImageInfo(); - List profile = (List) imageInfo.get("profile"); + Information iiifInfo = invokeNewImageInfo(); + List profile = (List) iiifInfo.get("profile"); // If some are present, we will assume the rest are. (The exact // contents of the sets are processor-dependent.) assertTrue(((Set) ((Map) profile.get(1)).get("qualities")).contains("color")); @@ -385,8 +385,8 @@ void testNewImageInfoMaxAreaWithPositiveMaxPixels() { final int maxPixels = 100; instance.setMaxPixels(maxPixels); - ImageInfo imageInfo = invokeNewImageInfo(); - List profile = (List) imageInfo.get("profile"); + Information iiifInfo = invokeNewImageInfo(); + List profile = (List) iiifInfo.get("profile"); assertEquals(maxPixels, ((Map) profile.get(1)).get("maxArea")); } @@ -395,8 +395,8 @@ void testNewImageInfoMaxAreaWithZeroMaxPixels() { final int maxPixels = 0; instance.setMaxPixels(maxPixels); - ImageInfo imageInfo = invokeNewImageInfo(); - List profile = (List) imageInfo.get("profile"); + Information iiifInfo = invokeNewImageInfo(); + List profile = (List) iiifInfo.get("profile"); assertFalse(((Map) profile.get(1)).containsKey("maxArea")); } @@ -406,16 +406,16 @@ void testNewImageInfoMaxAreaWithAllowUpscalingDisabled() { instance.setMaxPixels(maxPixels); instance.setMaxScale(1.0); - ImageInfo imageInfo = invokeNewImageInfo(); - List profile = (List) imageInfo.get("profile"); + Information iiifInfo = invokeNewImageInfo(); + List profile = (List) iiifInfo.get("profile"); assertEquals(1500 * 1200, ((Map) profile.get(1)).get("maxArea")); } @Test void testNewImageInfoSupports() { - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); - List profile = (List) imageInfo.get("profile"); + List profile = (List) iiifInfo.get("profile"); final Set supportsSet = (Set) ((Map) profile.get(1)).get("supports"); assertTrue(supportsSet.contains("baseUriRedirect")); assertTrue(supportsSet.contains("canonicalLinkHeader")); @@ -429,9 +429,9 @@ void testNewImageInfoSupports() { @Test void testNewImageInfoSupportsWhenUpscalingIsAllowed() { instance.setMaxScale(9.0); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); - List profile = (List) imageInfo.get("profile"); + List profile = (List) iiifInfo.get("profile"); final Set supportsSet = (Set) ((Map) profile.get(1)).get("supports"); assertTrue(supportsSet.contains("sizeAboveFull")); } @@ -439,9 +439,9 @@ void testNewImageInfoSupportsWhenUpscalingIsAllowed() { @Test void testNewImageInfoSupportsWhenUpscalingIsDisallowed() { instance.setMaxScale(1.0); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); - List profile = (List) imageInfo.get("profile"); + List profile = (List) iiifInfo.get("profile"); final Set supportsSet = (Set) ((Map) profile.get(1)).get("supports"); assertFalse(supportsSet.contains("sizeAboveFull")); } @@ -450,10 +450,10 @@ void testNewImageInfoSupportsWhenUpscalingIsDisallowed() { void testNewImageInfoSupportsWithScaleConstraint() { final String imageURI = "http://example.org/bla"; final Info info = Info.builder().withSize(1500, 1200).build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 4)); - List profile = (List) imageInfo.get("profile"); + List profile = (List) iiifInfo.get("profile"); final Set supportsSet = (Set) ((Map) profile.get(1)).get("supports"); assertFalse(supportsSet.contains("sizeAboveFull")); } @@ -463,12 +463,12 @@ void testNewImageInfoDelegateKeys() { DelegateProxy proxy = TestUtil.newDelegateProxy(); instance.setDelegateProxy(proxy); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); assertEquals("Copyright My Great Organization. All rights reserved.", - imageInfo.get("attribution")); + iiifInfo.get("attribution")); assertEquals("http://example.org/license.html", - imageInfo.get("license")); + iiifInfo.get("license")); } } diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResourceTest.java index 2b6d356bb..eac3cf0f3 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResourceTest.java @@ -440,7 +440,7 @@ void testGETURIsInJSON() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://localhost:" + HTTP_PORT + Route.IIIF_2_PATH + "/" + IMAGE, info.get("@id")); } @@ -455,7 +455,7 @@ void testGETURIsInJSONWithBaseURIOverride() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://example.org" + Route.IIIF_2_PATH + "/" + IMAGE, info.get("@id")); } @@ -471,7 +471,7 @@ void testGETURIsInJSONWithSlashSubstitution() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://localhost:" + HTTP_PORT + Route.IIIF_2_PATH + path, info.get("@id")); } @@ -487,7 +487,7 @@ void testGETURIsInJSONWithEncodedCharacters() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://localhost:" + HTTP_PORT + Route.IIIF_2_PATH + path, info.get("@id")); } @@ -505,7 +505,7 @@ void testGETURIsInJSONWithProxyHeaders() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://example.org:8080/cats" + Route.IIIF_2_PATH + "/originalID", info.get("@id")); } @@ -524,7 +524,7 @@ void testGETBaseURIOverridesProxyHeaders() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("https://example.net" + Route.IIIF_2_PATH + "/" + IMAGE, info.get("@id")); } diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/Version2_0ConformanceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/Version2_0ConformanceTest.java index 81714d12c..45c4bb997 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/Version2_0ConformanceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/Version2_0ConformanceTest.java @@ -558,7 +558,7 @@ void testInformationRequestCORSHeader() throws Exception { */ @Test void testInformationRequestJSON() { - // this will be tested in ImageInfoFactoryTest + // this will be tested in InformationFactoryTest } /** @@ -585,7 +585,7 @@ void testComplianceLevel() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); List profile = (List) info.get("profile"); assertEquals("http://iiif.io/api/image/2/level2.json", profile.get(0)); diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageInfoFactoryTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationFactoryTest.java similarity index 74% rename from src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageInfoFactoryTest.java rename to src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationFactoryTest.java index 77b750a30..7b187ba95 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageInfoFactoryTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationFactoryTest.java @@ -16,20 +16,20 @@ import static org.junit.jupiter.api.Assertions.*; -public class ImageInfoFactoryTest extends BaseTest { +public class InformationFactoryTest extends BaseTest { private static final Set PROCESSOR_FORMATS = Set.of(Format.get("gif"), Format.get("jpg"), Format.get("png")); - private ImageInfoFactory instance; + private InformationFactory instance; @BeforeEach public void setUp() throws Exception { super.setUp(); - instance = new ImageInfoFactory(); + instance = new InformationFactory(); } - private ImageInfo invokeNewImageInfo() { + private Information invokeNewImageInfo() { final String imageURI = "http://example.org/bla"; final Info info = Info.builder().withSize(1500, 1200).build(); return instance.newImageInfo(PROCESSOR_FORMATS, imageURI, info, 0, @@ -38,38 +38,38 @@ private ImageInfo invokeNewImageInfo() { @Test void testNewImageInfoContext() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals("http://iiif.io/api/image/3/context.json", info.get("@context")); } @Test void testNewImageInfoID() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals("http://example.org/bla", info.get("id")); } @Test void testNewImageInfoType() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals("ImageService3", info.get("type")); } @Test void testNewImageInfoProtocol() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals("http://iiif.io/api/image", info.get("protocol")); } @Test void testNewImageInfoProfile() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals("level2", info.get("profile")); } @Test void testNewImageInfoWidth() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals(1500, info.get("width")); } @@ -85,11 +85,11 @@ public Orientation getOrientation() { } }) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); - assertEquals(1200, imageInfo.get("width")); + assertEquals(1200, iiifInfo.get("width")); } @Test @@ -98,16 +98,16 @@ void testNewImageInfoWidthWithScaleConstrainedImage() { final Info info = Info.builder() .withSize(1499, 1199) // test rounding .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 2)); - assertEquals(750, imageInfo.get("width")); + assertEquals(750, iiifInfo.get("width")); } @Test void testNewImageInfoHeight() { - ImageInfo info = invokeNewImageInfo(); + Information info = invokeNewImageInfo(); assertEquals(1200, info.get("height")); } @@ -123,11 +123,11 @@ public Orientation getOrientation() { } }) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); - assertEquals(1500, imageInfo.get("height")); + assertEquals(1500, iiifInfo.get("height")); } @Test @@ -136,11 +136,11 @@ void testNewImageInfoHeightWithScaleConstrainedImage() { final Info info = Info.builder() .withSize(1499, 1199) // test rounding .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 2)); - assertEquals(600, imageInfo.get("height")); + assertEquals(600, iiifInfo.get("height")); } @Test @@ -150,8 +150,8 @@ void testNewImageInfoMaxAreaWithPositiveMaxPixelsGreaterThanAndPositiveMaxScale( instance.setMaxPixels(maxPixels); instance.setMaxScale(maxScale); - ImageInfo imageInfo = invokeNewImageInfo(); - assertEquals(maxPixels, imageInfo.get("maxArea")); + Information iiifInfo = invokeNewImageInfo(); + assertEquals(maxPixels, iiifInfo.get("maxArea")); } @Test @@ -161,8 +161,8 @@ void testNewImageInfoMaxAreaWithPositiveMaxPixelsLessThanPositiveMaxScale() { instance.setMaxPixels(maxPixels); instance.setMaxScale(maxScale); - ImageInfo imageInfo = invokeNewImageInfo(); - assertEquals(maxPixels, imageInfo.get("maxArea")); // TODO: fix + Information iiifInfo = invokeNewImageInfo(); + assertEquals(maxPixels, iiifInfo.get("maxArea")); // TODO: fix } @Test @@ -170,8 +170,8 @@ void testNewImageInfoMaxAreaWithPositiveMaxPixelsAndZeroMaxScale() { final long maxPixels = 100; instance.setMaxPixels(maxPixels); - ImageInfo imageInfo = invokeNewImageInfo(); - assertEquals(maxPixels, imageInfo.get("maxArea")); + Information iiifInfo = invokeNewImageInfo(); + assertEquals(maxPixels, iiifInfo.get("maxArea")); } @Test @@ -180,9 +180,9 @@ void testNewImageInfoMaxAreaWithZeroMaxPixelsAndPositiveMaxScale() { instance.setMaxPixels(0); instance.setMaxScale(maxScale); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); assertEquals(Math.round(1500 * 1200 * maxScale), - imageInfo.get("maxArea")); + iiifInfo.get("maxArea")); } @Test @@ -191,8 +191,8 @@ void testNewImageInfoMaxAreaWithZeroMaxPixelsAndZeroMaxScale() { instance.setMaxScale(0); instance.setMaxPixels(maxPixels); - ImageInfo imageInfo = invokeNewImageInfo(); - assertFalse(imageInfo.containsKey("maxArea")); + Information iiifInfo = invokeNewImageInfo(); + assertFalse(iiifInfo.containsKey("maxArea")); } @Test @@ -201,17 +201,17 @@ void testNewImageInfoMaxAreaWithAllowUpscalingDisabled() { instance.setMaxPixels(maxPixels); instance.setMaxScale(1); - ImageInfo imageInfo = invokeNewImageInfo(); - assertEquals((long) Math.round(1500 * 1200), imageInfo.get("maxArea")); + Information iiifInfo = invokeNewImageInfo(); + assertEquals((long) Math.round(1500 * 1200), iiifInfo.get("maxArea")); } @Test void testNewImageInfoSizes() { - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); @SuppressWarnings("unchecked") - List sizes = - (List) imageInfo.get("sizes"); + List sizes = + (List) iiifInfo.get("sizes"); assertEquals(5, sizes.size()); assertEquals(94, (int) sizes.get(0).width); assertEquals(75, (int) sizes.get(0).height); @@ -228,11 +228,11 @@ void testNewImageInfoSizes() { @Test void testNewImageInfoSizesMinSize() { instance.setMinSize(500); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); @SuppressWarnings("unchecked") - List sizes = - (List) imageInfo.get("sizes"); + List sizes = + (List) iiifInfo.get("sizes"); assertEquals(2, sizes.size()); assertEquals(750, (int) sizes.get(0).width); assertEquals(600, (int) sizes.get(0).height); @@ -243,11 +243,11 @@ void testNewImageInfoSizesMinSize() { @Test void testNewImageInfoSizesMaxSize() { instance.setMaxPixels(10000); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); @SuppressWarnings("unchecked") - List sizes = - (List) imageInfo.get("sizes"); + List sizes = + (List) iiifInfo.get("sizes"); assertEquals(1, sizes.size()); assertEquals(94, (int) sizes.get(0).width); assertEquals(75, (int) sizes.get(0).height); @@ -265,13 +265,13 @@ public Orientation getOrientation() { } }) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); @SuppressWarnings("unchecked") - List sizes = - (List) imageInfo.get("sizes"); + List sizes = + (List) iiifInfo.get("sizes"); assertEquals(5, sizes.size()); assertEquals(75, (int) sizes.get(0).width); assertEquals(94, (int) sizes.get(0).height); @@ -291,13 +291,13 @@ void testNewImageInfoSizesWithScaleConstrainedImage() { final Info info = Info.builder() .withSize(1500, 1200) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 2)); @SuppressWarnings("unchecked") - List sizes = - (List) imageInfo.get("sizes"); + List sizes = + (List) iiifInfo.get("sizes"); assertEquals(4, sizes.size()); assertEquals(94, (int) sizes.get(0).width); assertEquals(75, (int) sizes.get(0).height); @@ -311,11 +311,11 @@ void testNewImageInfoSizesWithScaleConstrainedImage() { @Test void testNewImageInfoTilesWithUntiledMonoResolutionImage() { - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(1, tiles.size()); assertEquals(512, (int) tiles.get(0).width); assertEquals(512, (int) tiles.get(0).height); @@ -335,13 +335,13 @@ void testNewImageInfoTilesWithUntiledMultiResolutionImage() { .withSize(3000, 2000) .withNumResolutions(3) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(1, tiles.size()); assertEquals(512, (int) tiles.get(0).width); assertEquals(512, (int) tiles.get(0).height); @@ -363,13 +363,13 @@ void testNewImageInfoMinTileSize() { .withTileSize(1000, 1000) .build(); instance.setMinTileSize(1000); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(1000, (int) tiles.get(0).width); assertEquals(1000, (int) tiles.get(0).height); } @@ -387,13 +387,13 @@ public Orientation getOrientation() { }) .withTileSize(64, 56) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(56, (int) tiles.get(0).width); assertEquals(64, (int) tiles.get(0).height); } @@ -405,13 +405,13 @@ void testNewImageInfoTilesWithScaleConstrainedImage() { .withSize(64, 56) .withTileSize(64, 56) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 2)); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(32, (int) tiles.get(0).width); assertEquals(28, (int) tiles.get(0).height); } @@ -423,13 +423,13 @@ void testNewImageInfoTilesWithTiledImage() { .withSize(64, 56) .withTileSize(64, 56) .build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 1)); @SuppressWarnings("unchecked") - List tiles = - (List) imageInfo.get("tiles"); + List tiles = + (List) iiifInfo.get("tiles"); assertEquals(1, tiles.size()); assertEquals(64, (int) tiles.get(0).width); assertEquals(56, (int) tiles.get(0).height); @@ -440,8 +440,8 @@ void testNewImageInfoTilesWithTiledImage() { @Test void testNewImageInfoExtraQualities() { - ImageInfo imageInfo = invokeNewImageInfo(); - List qualities = (List) imageInfo.get("extraQualities"); + Information iiifInfo = invokeNewImageInfo(); + List qualities = (List) iiifInfo.get("extraQualities"); assertEquals(3, qualities.size()); assertTrue(qualities.contains("color")); assertTrue(qualities.contains("gray")); @@ -450,25 +450,25 @@ void testNewImageInfoExtraQualities() { @Test void testNewImageInfoExtraFormats() { - ImageInfo imageInfo = invokeNewImageInfo(); - List formats = (List) imageInfo.get("extraFormats"); + Information iiifInfo = invokeNewImageInfo(); + List formats = (List) iiifInfo.get("extraFormats"); assertEquals(1, formats.size()); assertTrue(formats.contains("gif")); } @Test void testNewImageInfoExtraFeatures() { - ImageInfo imageInfo = invokeNewImageInfo(); - List features = (List) imageInfo.get("extraFeatures"); + Information iiifInfo = invokeNewImageInfo(); + List features = (List) iiifInfo.get("extraFeatures"); assertEquals(17, features.size()); } @Test void testNewImageInfoExtraFeaturesOmitsSizeUpscalingWhenMaxScaleIsLessThanOrEqualTo1() { instance.setMaxScale(1.0); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); - List features = (List) imageInfo.get("extraFeatures"); + List features = (List) iiifInfo.get("extraFeatures"); assertEquals(16, features.size()); assertFalse(features.contains("sizeUpscaling")); } @@ -476,18 +476,18 @@ void testNewImageInfoExtraFeaturesOmitsSizeUpscalingWhenMaxScaleIsLessThanOrEqua @Test void testNewImageInfoExtraFeaturesWhenUpscalingIsAllowed() { instance.setMaxScale(9.0); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); - List features = (List) imageInfo.get("extraFeatures"); + List features = (List) iiifInfo.get("extraFeatures"); assertTrue(features.contains("sizeUpscaling")); } @Test void testNewImageInfoExtraFeaturesWhenUpscalingIsDisallowed() { instance.setMaxScale(1.0); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); - List features = (List) imageInfo.get("extraFeatures"); + List features = (List) iiifInfo.get("extraFeatures"); assertFalse(features.contains("sizeUpscaling")); } @@ -495,11 +495,11 @@ void testNewImageInfoExtraFeaturesWhenUpscalingIsDisallowed() { void testNewImageInfoExtraFeaturesWithScaleConstraint() { final String imageURI = "http://example.org/bla"; final Info info = Info.builder().withSize(1500, 1200).build(); - ImageInfo imageInfo = instance.newImageInfo( + Information iiifInfo = instance.newImageInfo( PROCESSOR_FORMATS, imageURI, info, 0, new ScaleConstraint(1, 4)); - List features = (List) imageInfo.get("extraFeatures"); + List features = (List) iiifInfo.get("extraFeatures"); assertFalse(features.contains("sizeUpscaling")); } @@ -508,12 +508,12 @@ void testNewImageInfoDelegateKeys() { DelegateProxy proxy = TestUtil.newDelegateProxy(); instance.setDelegateProxy(proxy); - ImageInfo imageInfo = invokeNewImageInfo(); + Information iiifInfo = invokeNewImageInfo(); assertEquals("Copyright My Great Organization. All rights reserved.", - imageInfo.get("attribution")); + iiifInfo.get("attribution")); assertEquals("http://example.org/license.html", - imageInfo.get("license")); + iiifInfo.get("license")); } } diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResourceTest.java index d361a1f8c..d704eb4f7 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResourceTest.java @@ -415,7 +415,7 @@ void testGETURIsInJSON() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://localhost:" + HTTP_PORT + Route.IIIF_3_PATH + "/" + IMAGE, info.get("id")); } @@ -430,7 +430,7 @@ void testGETURIsInJSONWithBaseURIOverride() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://example.org" + Route.IIIF_3_PATH + "/" + IMAGE, info.get("id")); } @@ -446,7 +446,7 @@ void testGETURIsInJSONWithSlashSubstitution() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://localhost:" + HTTP_PORT + Route.IIIF_3_PATH + path, info.get("id")); } @@ -462,7 +462,7 @@ void testGETURIsInJSONWithEncodedCharacters() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://localhost:" + HTTP_PORT + Route.IIIF_3_PATH + path, info.get("id")); } @@ -480,7 +480,7 @@ void testGETURIsInJSONWithProxyHeaders() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("http://example.org:8080/cats" + Route.IIIF_3_PATH + "/originalID", info.get("id")); } @@ -499,7 +499,7 @@ void testGETBaseURIOverridesProxyHeaders() throws Exception { String json = response.getBodyAsString(); ObjectMapper mapper = new ObjectMapper(); - ImageInfo info = mapper.readValue(json, ImageInfo.class); + Information info = mapper.readValue(json, Information.class); assertEquals("https://example.net" + Route.IIIF_3_PATH + "/" + IMAGE, info.get("id")); } diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/Version3_0ConformanceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/Version3_0ConformanceTest.java index 7242b172f..a4b4c2590 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/Version3_0ConformanceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/Version3_0ConformanceTest.java @@ -882,7 +882,7 @@ void testInformationRequestCORSHeader() throws Exception { */ @Test void testInformationRequestJSON() { - // this is tested in ImageInfoFactoryTest + // this is tested in InformationFactoryTest } } From 3d1e3616a18ee0274e435460fd3af933b390606d Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Fri, 28 May 2021 10:24:53 -0500 Subject: [PATCH 002/106] Support a cantaloupe.delegate_script VM argument --- CHANGES.md | 6 + cantaloupe.properties.sample | 2 + .../delegate/DelegateProxyService.java | 33 +++-- src/main/resources/admin.vm | 4 +- .../delegate/DelegateProxyServiceTest.java | 124 ++++++++++++++---- 5 files changed, 128 insertions(+), 41 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index cf843b5f5..100733a71 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,11 @@ # Change Log +# 6.0 + +* The delegate script pathname can be set using the + `-Dcantaloupe.delegate_script` VM argument, which takes precedence over the + `delegate_script.pathname` configuration key. + ## 5.0.3 * Suppressed an error-level log message from OpenJpegProcessor when reading diff --git a/cantaloupe.properties.sample b/cantaloupe.properties.sample index 6de429746..0f89ccb5f 100644 --- a/cantaloupe.properties.sample +++ b/cantaloupe.properties.sample @@ -88,6 +88,8 @@ delegate_script.enabled = false # !! This can be an absolute path, or a filename; if only a filename is # specified, it will be searched for in the same folder as this file, and # then the current working directory. +# The delegate script pathname can also be set using the +# -Dcantaloupe.delegate_script VM argument, which overrides this value. delegate_script.pathname = delegates.rb ########################################################################### diff --git a/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateProxyService.java b/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateProxyService.java index 90ed0ee46..6c9c91c14 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateProxyService.java +++ b/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateProxyService.java @@ -26,6 +26,9 @@ public final class DelegateProxyService { private static final Logger LOGGER = LoggerFactory.getLogger(DelegateProxyService.class); + static final String DELEGATE_SCRIPT_VM_ARGUMENT = + "cantaloupe.delegate_script"; + private static DelegateProxyService instance; private static boolean isScriptCodeLoaded; @@ -95,23 +98,29 @@ public static synchronized DelegateProxyService getInstance() { } /** - * @return Absolute path representing the delegate script, regardless of - * whether the delegate script system is {@link #isScriptEnabled() - * enabled}; or {@code null} if {@link - * Key#DELEGATE_SCRIPT_PATHNAME} is not set. + *

Returns the absolute path to the delegate script, regardless of + * whether the delegate script system is {@link #isScriptEnabled() + * enabled}. The path is obtained from the {@link + * #DELEGATE_SCRIPT_VM_ARGUMENT delegate script VM argument}, if set, or + * the {@link Key#DELEGATE_SCRIPT_PATHNAME configuration} otherwise. If + * neither are set, set, {@code null} is returned.

+ * + *

The contents of the script are not validated.

+ * * @throws NoSuchFileException If the script specified in {@link * Key#DELEGATE_SCRIPT_PATHNAME} does not exist. */ static Path getScriptFile() throws NoSuchFileException { - final Configuration config = Configuration.getInstance(); - // The script name may be an absolute pathname or a filename. - final String configValue = - config.getString(Key.DELEGATE_SCRIPT_PATHNAME, ""); - if (!configValue.isEmpty()) { - Path script = findScript(configValue); + String value = System.getProperty("cantaloupe.delegate_script"); + if (value == null || value.isBlank()) { + final Configuration config = Configuration.getInstance(); + // The script name may be an absolute pathname or a filename. + value = config.getString(Key.DELEGATE_SCRIPT_PATHNAME, ""); + } + if (!value.isBlank()) { + Path script = findScript(value); if (!Files.exists(script)) { - throw new NoSuchFileException( - "File not found: " + script.toString()); + throw new NoSuchFileException("File not found: " + script); } return script; } diff --git a/src/main/resources/admin.vm b/src/main/resources/admin.vm index 134f11b17..89e852b9d 100644 --- a/src/main/resources/admin.vm +++ b/src/main/resources/admin.vm @@ -238,7 +238,9 @@ data-content="Location of the delegate script. This can be an absolute path, or a filename; if only a filename is specified, it will be searched for in the same folder as the config file, - and then the current working directory.">? + and then the current working directory. The delegate script + pathname can also be set using the <code>-Dcantaloupe.delegate_script</code> + VM argument, which overrides this value.">? Date: Fri, 28 May 2021 14:33:56 -0500 Subject: [PATCH 003/106] HttpSource supports ranged GET requests instead of HEAD requests (#465) --- CHANGES.md | 3 + UPGRADING.md | 5 + cantaloupe.properties.sample | 6 + delegates.rb.sample | 15 +- .../library/cantaloupe/config/Key.java | 1 + .../cantaloupe/delegate/JavaDelegate.java | 5 + .../cantaloupe/source/HTTPRequestInfo.java | 51 ++- .../library/cantaloupe/source/HttpSource.java | 419 +++++++++--------- .../OkHttpHTTPImageInputStreamClient.java | 4 +- src/main/resources/admin.vm | 31 +- .../resource/admin/AdminResourceUITest.java | 3 + .../source/HTTPRequestInfoTest.java | 36 +- .../source/HTTPStreamFactoryTest.java | 10 +- .../cantaloupe/source/HttpSourceTest.java | 106 +++-- src/test/resources/delegates.rb | 12 +- 15 files changed, 420 insertions(+), 287 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 100733a71..a86402fc1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,9 @@ # 6.0 +* HttpSource can be configured to send a ranged GET request instead of a HEAD + request, enabling it to work with pre-signed URLs that do not allow HEAD + requests. * The delegate script pathname can be set using the `-Dcantaloupe.delegate_script` VM argument, which takes precedence over the `delegate_script.pathname` configuration key. diff --git a/UPGRADING.md b/UPGRADING.md index db1e9a7e3..65a474b7e 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -3,6 +3,11 @@ If you are skipping versions, work through these sections backwards from your current version. +## 5.0 → 6.0 + +1. Add the following keys from the sample configuration: + * `HttpSource.BasicLookupStrategy.send_head_requests` + ## 4.1.x → 5.0 1. Note that the application is now packaged as a JAR file which can no longer diff --git a/cantaloupe.properties.sample b/cantaloupe.properties.sample index 0f89ccb5f..3d6248650 100644 --- a/cantaloupe.properties.sample +++ b/cantaloupe.properties.sample @@ -189,6 +189,12 @@ HttpSource.BasicLookupStrategy.url_suffix = HttpSource.BasicLookupStrategy.auth.basic.username = HttpSource.BasicLookupStrategy.auth.basic.secret = +# Before an image is retrieved, a preliminary request is sent to check +# various characteristics. Typically this is a HEAD request, but some +# resources, such as those using pre-signed URLs, may not support HEAD +# requests. This key enables a ranged GET request to be sent instead. +HttpSource.BasicLookupStrategy.send_head_requests = true + # Read data in chunks when it may be more efficient. (This also may end up # being less efficient, depending on many variables; see the user manual.) HttpSource.chunking.enabled = true diff --git a/delegates.rb.sample b/delegates.rb.sample index ea9c5f4ba..04ac612c3 100644 --- a/delegates.rb.sample +++ b/delegates.rb.sample @@ -221,10 +221,17 @@ class CustomDelegate # # 1. String URI # 2. Hash with the following keys: - # * `uri` [String] (required) - # * `username` [String] For HTTP Basic authentication (optional). - # * `secret` [String] For HTTP Basic authentication (optional). - # * `headers` [Hash] Hash of request headers (optional). + # * `uri` [String] (required) + # * `username` [String] For HTTP Basic authentication + # (optional). + # * `secret` [String] For HTTP Basic authentication + # (optional). + # * `headers` [Hash] Hash of request headers + # (optional). + # * `send_head_request` [Boolean] Optional; defaults to `true`. See the + # documentation of the + # `HttpSource.BasicLookupStrategy.send_head_requests` + # configuration key. # 3. nil if not found. # # N.B.: this method should not try to perform authorization. `authorize()` diff --git a/src/main/java/edu/illinois/library/cantaloupe/config/Key.java b/src/main/java/edu/illinois/library/cantaloupe/config/Key.java index 6ecab5cf9..3f53ff0af 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/config/Key.java +++ b/src/main/java/edu/illinois/library/cantaloupe/config/Key.java @@ -109,6 +109,7 @@ public enum Key { HTTPSOURCE_CHUNK_CACHE_MAX_SIZE("HttpSource.chunking.cache.max_size"), HTTPSOURCE_LOOKUP_STRATEGY("HttpSource.lookup_strategy"), HTTPSOURCE_REQUEST_TIMEOUT("HttpSource.request_timeout"), + HTTPSOURCE_SEND_HEAD_REQUESTS("HttpSource.BasicLookupStrategy.send_head_requests"), HTTPSOURCE_URL_PREFIX("HttpSource.BasicLookupStrategy.url_prefix"), HTTPSOURCE_URL_SUFFIX("HttpSource.BasicLookupStrategy.url_suffix"), HTTPS_ENABLED("https.enabled"), diff --git a/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaDelegate.java b/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaDelegate.java index cba44062f..7a4b3f54e 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaDelegate.java +++ b/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaDelegate.java @@ -179,6 +179,11 @@ public interface JavaDelegate { *
String (required for HTTP Basic authentication)
*
{@code headers}
*
Map of request header name-value pairs (optional)
+ *
{@code send_head_request}
+ *
Boolean (optional). Defaults to {@code true}. See the + * documentation of the {@code + * HttpSource.BasicLookupStrategy.send_head_requests} + * configuration key.
* * *
  • {@code null} if not found
    • diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/HTTPRequestInfo.java b/src/main/java/edu/illinois/library/cantaloupe/source/HTTPRequestInfo.java index fa3d2033d..1b4bebc0d 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/HTTPRequestInfo.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/HTTPRequestInfo.java @@ -10,27 +10,7 @@ class HTTPRequestInfo { private final Headers headers = new Headers(); private String uri, username, secret; - - HTTPRequestInfo(String uri) { - this.uri = uri; - } - - HTTPRequestInfo(String uri, String username, String secret) { - this(uri); - this.username = username; - this.secret = secret; - } - - HTTPRequestInfo(String uri, - String username, - String secret, - Map headers) { - this(uri, username, secret); - if (headers != null) { - headers.forEach((key, value) -> - this.headers.add(key, value.toString())); - } - } + private boolean isSendingHeadRequest = true; String getBasicAuthToken() { if (getUsername() != null && getSecret() != null) { @@ -57,6 +37,35 @@ String getUsername() { return username; } + boolean isSendingHeadRequest() { + return isSendingHeadRequest; + } + + void setHeaders(Map headers) { + if (headers != null) { + headers.forEach((key, value) -> + this.headers.add(key, value.toString())); + } else { + this.headers.clear(); + } + } + + void setSecret(String secret) { + this.secret = secret; + } + + void setSendingHeadRequest(boolean isUsingHeadRequest) { + this.isSendingHeadRequest = isUsingHeadRequest; + } + + void setURI(String uri) { + this.uri = uri; + } + + void setUsername(String username) { + this.username = username; + } + @Override public String toString() { return getURI() + ""; diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java b/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java index fe39a6c7f..c4c38ade3 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java @@ -65,12 +65,16 @@ * issues the following server requests:

    * *
      - *
    1. {@literal HEAD}
      2. - *
    2. If server supports ranges: + *
    3. If {@link Key#HTTPSOURCE_SEND_HEAD_REQUESTS} is {@code true}, or + * the delegate method returns {@code true} for the equivalent key, a + * {@code HEAD} request. Otherwise, a ranged {@code GET} request specifying + * a small range of the beginning of the resource.
      4. + *
    4. If a {@code HEAD} request was sent: *
        - *
      1. If {@link FormatIterator#next()} } needs to check magic + *
      2. If {@link FormatIterator#next()} needs to check magic bytes, + * and the server supports ranges: *
          - *
        1. Ranged {@literal GET}
          2. + *
        2. Ranged {@code GET}
          3. *
        *
        3. *
      3. If {@link HTTPStreamFactory#newSeekableStream()} is used: @@ -82,7 +86,7 @@ *
        4. *
      4. Else if {@link HTTPStreamFactory#newInputStream()} is used: *
          - *
        1. {@literal GET} to retrieve the full image bytes
          2. + *
        2. {@code GET} to retrieve the full image bytes
          3. *
        *
        5. *
      @@ -111,18 +115,31 @@ class HttpSource extends AbstractSource implements Source { /** - * Encapsulates the status code and headers of a HEAD response. + * Encapsulates the status code, headers, and body (if available) of a + * {code HEAD} or ranged {@code GET} response. The range specifies a small + * part of the beginning of the resource to use for the purpose of + * inferring its format. */ - private static class HEADResponseInfo { + private static class ResourceInfo { - int status; - Headers headers; + private String requestMethod; + private int status; + private Headers headers; - static HEADResponseInfo fromResponse(Response response) - throws IOException { - HEADResponseInfo info = new HEADResponseInfo(); - info.status = response.code(); - info.headers = response.headers(); + /** + * Response entity with a maximum length of {@link #RANGE_LENGTH}. + */ + private byte[] entity; + + static ResourceInfo fromResponse(Response response) throws IOException { + ResourceInfo info = new ResourceInfo(); + info.requestMethod = response.request().method(); + info.status = response.code(); + info.headers = response.headers(); + if ("GET".equals(response.request().method()) && + response.body() != null) { + info.entity = response.body().bytes(); + } return info; } @@ -130,37 +147,13 @@ boolean acceptsRanges() { return "bytes".equals(headers.get("Accept-Ranges")); } - long getContentLength() { + long contentLength() { String value = headers.get("Content-Length"); return (value != null) ? Long.parseLong(value) : 0; } - } - - /** - * Encapsulates the status code, headers, and body of a ranged GET - * response. The range specifies a small part of the beginning of the - * resource to use for the purpose of inferring its format. - */ - private static class RangedGETResponseInfo extends HEADResponseInfo { - - private static final int RANGE_LENGTH = 32; - - /** - * Ranged response entity, with a maximum length of {@link - * #RANGE_LENGTH}. - */ - private byte[] entity; - - static RangedGETResponseInfo fromResponse(Response response) - throws IOException { - RangedGETResponseInfo info = new RangedGETResponseInfo(); - info.status = response.code(); - info.headers = response.headers(); - if (response.body() != null) { - info.entity = response.body().bytes(); - } - return info; + String contentType() { + return headers.get("Content-Type"); } Format detectFormat() throws IOException { @@ -182,13 +175,13 @@ Format detectFormat() throws IOException { * extension, the format is inferred from that.
      5. *
    5. Otherwise, if the identifier contains a recognized filename * extension, the format is inferred from that.
      6. - *
    6. Otherwise, if a {@literal Content-Type} header is present in the - * {@link #fetchHEADResponseInfo() HEAD response}, and its value is - * specific enough (not {@literal application/octet-stream}, for + *
    7. Otherwise, if a {@code Content-Type} header is present in the + * {@link #getResourceInfo HEAD response}, and its value is + * specific enough (not {@code application/octet-stream}, for * example), a format is inferred from that.
      8. - *
    8. Otherwise, if the {@literal HEAD} response contains an {@literal - * Accept-Ranges: bytes} header, a {@literal GET} request is sent with - * a {@literal Range} header specifying a small range of data from the + *
    9. Otherwise, if the {@literal HEAD} response contains an {@code + * Accept-Ranges: bytes} header, a {@code GET} request is sent with a + * {@code Range} header specifying a small range of data from the * beginning of the resource, and a format is inferred from the magic * bytes in the response entity.
      10. *
    10. Otherwise, {@link Format#UNKNOWN} is returned.
      11. @@ -199,36 +192,40 @@ Format detectFormat() throws IOException { class FormatIterator implements Iterator { /** - * Infers a {@link Format} based on the {@literal Content-Type} header in - * the {@link #fetchHEADResponseInfo() HEAD response}. + * Infers a {@link Format} based on the {@code Content-Type} header in + * the initial response.. */ private class ContentTypeHeaderChecker implements FormatChecker { /** - * @return Format from the {@literal Content-Type} header, or {@link + * @return Format from the {@code Content-Type} header, or {@link * Format#UNKNOWN} if that header is missing or invalid. */ @Override public Format check() { try { final HTTPRequestInfo requestInfo = getRequestInfo(); - final HEADResponseInfo responseInfo = fetchHEADResponseInfo(); + final ResourceInfo resourceInfo = getResourceInfo(); - if (responseInfo.status >= 200 && responseInfo.status < 300) { - String field = responseInfo.headers.get("Content-Type"); - if (field != null) { - Format format = MediaType.fromContentType(field).toFormat(); + if (resourceInfo.status >= 200 && resourceInfo.status < 300) { + String value = resourceInfo.contentType(); + if (value != null) { + Format format = MediaType.fromContentType(value).toFormat(); if (Format.UNKNOWN.equals(format)) { - LOGGER.debug("Unrecognized Content-Type header value for HEAD {}", + LOGGER.debug("Unrecognized Content-Type header value for {} {}", + resourceInfo.requestMethod, requestInfo.getURI()); } return format; } else { - LOGGER.debug("No Content-Type header for HEAD {}", + LOGGER.debug("No Content-Type header for {} {}", + resourceInfo.requestMethod, requestInfo.getURI()); } } else { - LOGGER.debug("HEAD {} returned status {}", - requestInfo.getURI(), responseInfo.status); + LOGGER.debug("{} {} returned status {}", + resourceInfo.requestMethod, + requestInfo.getURI(), + resourceInfo.status); } } catch (Exception e) { LOGGER.error(e.getMessage(), e); @@ -239,10 +236,12 @@ public Format check() { private class ByteChecker implements FormatChecker { /** - * If the {@link #fetchHEADResponseInfo() HEAD response} contains an - * {@literal Accept-Ranges: bytes} header, issues an HTTP {@literal GET} - * request for a small {@literal Range} of the beginning of the resource - * and checks the magic bytes in the response body. + * If the {@link #getResourceInfo initial response} is from a + * {@code HEAD} request, issues an HTTP {@code GET} request for a + * small range of the beginning of the resource. (If it is a {@code + * GET} response, that data has already been received.) Then, a + * source format is inferred from the magic bytes in the response + * entity. * * @return Inferred source format, or {@link Format#UNKNOWN}. */ @@ -250,28 +249,43 @@ private class ByteChecker implements FormatChecker { public Format check() { try { final HTTPRequestInfo requestInfo = getRequestInfo(); - if (fetchHEADResponseInfo().acceptsRanges()) { - final RangedGETResponseInfo responseInfo - = fetchRangedGETResponseInfo(); - if (responseInfo.status >= 200 && responseInfo.status < 300) { - Format format = responseInfo.detectFormat(); - if (!Format.UNKNOWN.equals(format)) { - LOGGER.debug("Inferred {} format from magic bytes for GET {}", - format, requestInfo.getURI()); - return format; - } else { - LOGGER.debug("Unable to infer a format from magic bytes for GET {}", - requestInfo.getURI()); - } + // If a request hasn't yet been sent, send one. This may be + // a HEAD or a ranged GET. It's not safe to send a ranged + // GET without first checking (via HEAD) whether the + // resource supports ranges--unless we are told via the + // configuration not to send HEADs. + if (resourceInfo == null) { + resourceInfo = getResourceInfo(); + } + // If it was a HEAD, we need to know whether the resource + // supports ranged requests. If it does, send one. + if ("HEAD".equals(resourceInfo.requestMethod)) { + if (resourceInfo.acceptsRanges()) { + resourceInfo = fetchResourceInfoViaGET(); + } else { + LOGGER.debug("Server did not supply an " + + "`Accept-Ranges: bytes` header in response " + + "to HEAD {}, and all other attempts to "+ + "infer a format failed.", + requestInfo.getURI()); + return Format.UNKNOWN; + } + } + if (resourceInfo.status >= 200 && resourceInfo.status < 300) { + Format format = resourceInfo.detectFormat(); + if (!Format.UNKNOWN.equals(format)) { + LOGGER.debug("Inferred {} format from magic bytes for GET {}", + format, requestInfo.getURI()); + return format; } else { - LOGGER.debug("GET {} returned status {}", - requestInfo.getURI(), responseInfo.status); + LOGGER.debug("Unable to infer a format from magic bytes for GET {}", + requestInfo.getURI()); } } else { - LOGGER.info("Server did not supply an " + - "`Accept-Ranges: bytes` header for HEAD {}, and all " + - "other attempts to infer a format failed.", - requestInfo.getURI()); + LOGGER.debug("{} {} returned status {}", + resourceInfo.requestMethod, + requestInfo.getURI(), + resourceInfo.status); } } catch (Exception e) { LOGGER.error(e.getMessage(), e); @@ -335,27 +349,39 @@ public Format check() { static final Logger LOGGER = LoggerFactory.getLogger(HttpSource.class); + static final String USER_AGENT = String.format( + "%s/%s (%s/%s; java/%s; %s/%s)", + HttpSource.class.getSimpleName(), + Application.getVersion(), + Application.getName(), + Application.getVersion(), + System.getProperty("java.version"), + System.getProperty("os.name"), + System.getProperty("os.version")); + private static final int DEFAULT_REQUEST_TIMEOUT = 30; + private static final int RANGE_LENGTH = 32; private static OkHttpClient httpClient; /** - * Cached {@link #fetchHEADResponseInfo() HEAD response info}. + * Cached by {@link #getRequestInfo()}. */ - private HEADResponseInfo headResponseInfo; + private HTTPRequestInfo requestInfo; /** - * Cached {@link #fetchRangedGETResponseInfo() ranged GET response info}. + * Cached {@link #getResourceInfo resource info} from the initial request, + * which may be either a {@code HEAD} or ranged {@code GET}, depending on + * the configuration. */ - private RangedGETResponseInfo rangedGETResponseInfo; + private ResourceInfo resourceInfo; + + private final FormatIterator formatIterator = + new FormatIterator<>(); /** - * Cached by {@link #getRequestInfo()}. + * @return Already-initialized instance shared by all threads. */ - private HTTPRequestInfo requestInfo; - - private FormatIterator formatIterator = new FormatIterator<>(); - static synchronized OkHttpClient getHTTPClient() { if (httpClient == null) { final OkHttpClient.Builder builder = new OkHttpClient.Builder() @@ -408,22 +434,10 @@ private static Duration getRequestTimeout() { return Duration.ofSeconds(timeout); } - static String getUserAgent() { - return String.format("%s/%s (%s/%s; java/%s; %s/%s)", - HttpSource.class.getSimpleName(), - Application.getVersion(), - Application.getName(), - Application.getVersion(), - System.getProperty("java.version"), - System.getProperty("os.name"), - System.getProperty("os.version")); - } - @Override public void checkAccess() throws IOException { - fetchHEADResponseInfo(); - - final int status = headResponseInfo.status; + ResourceInfo info = getResourceInfo(); + final int status = info.status; if (status >= 400) { final String statusLine = "HTTP " + status; if (status == 404 || status == 410) { // not found or gone @@ -441,95 +455,73 @@ public FormatIterator getFormatIterator() { return formatIterator; } - @Override - public StreamFactory newStreamFactory() throws IOException { - HTTPRequestInfo info; - try { - info = getRequestInfo(); - } catch (IOException e) { - throw e; - } catch (Exception e) { - LOGGER.error("newStreamFactory(): {}", e.getMessage()); - throw new IOException(e); - } - - if (info != null) { - LOGGER.debug("Resolved {} to {}", identifier, info.getURI()); - fetchHEADResponseInfo(); - return new HTTPStreamFactory( - getHTTPClient(), - info, - headResponseInfo.getContentLength(), - headResponseInfo.acceptsRanges()); - } - return null; - } - /** - *

      Issues a {@literal HEAD} request and caches parts of the response in - * {@link #headResponseInfo}.

      + * Issues a {@code HEAD} or ranged {@code GET} request (depending on the + * configuration) and caches the result in {@link #resourceInfo}. */ - private HEADResponseInfo fetchHEADResponseInfo() throws IOException { - if (headResponseInfo == null) { - try (Response response = request("HEAD")) { - headResponseInfo = HEADResponseInfo.fromResponse(response); + private ResourceInfo getResourceInfo() throws IOException { + if (resourceInfo == null) { + try { + requestInfo = getRequestInfo(); + if (requestInfo.isSendingHeadRequest()) { + fetchResourceInfoViaHEAD(); + } else { + fetchResourceInfoViaGET(); + } + } catch (Exception e) { + LOGGER.error("fetchResourceInfo(): {}", e.getMessage()); + throw new IOException(e.getMessage(), e); } } - return headResponseInfo; + return resourceInfo; } - /** - *

      Issues a {@literal GET} request specifying a small range of data and - * caches parts of the response in {@link #rangedGETResponseInfo}.

      - */ - private RangedGETResponseInfo fetchRangedGETResponseInfo() - throws IOException { - if (rangedGETResponseInfo == null) { - Map extraHeaders = Map.of("Range", - "bytes=0-" + (RangedGETResponseInfo.RANGE_LENGTH - 1)); - try (Response response = request("GET", extraHeaders)) { - rangedGETResponseInfo = - RangedGETResponseInfo.fromResponse(response); - } + private ResourceInfo fetchResourceInfoViaHEAD() throws Exception { + requestInfo = getRequestInfo(); + try (Response response = request("HEAD", Collections.emptyMap())) { + resourceInfo = ResourceInfo.fromResponse(response); } - return rangedGETResponseInfo; + return resourceInfo; } - private Response request(String method) throws IOException { - return request(method, Collections.emptyMap()); + private ResourceInfo fetchResourceInfoViaGET() throws Exception { + requestInfo = getRequestInfo(); + var extraHeaders = Map.of("Range", "bytes=0-" + (RANGE_LENGTH - 1)); + try (Response response = request("GET", extraHeaders)) { + resourceInfo = ResourceInfo.fromResponse(response); + } + return resourceInfo; } private Response request(String method, Map extraHeaders) throws IOException { - HTTPRequestInfo requestInfo; try { - requestInfo = getRequestInfo(); + HTTPRequestInfo requestInfo = getRequestInfo(); + + Request.Builder builder = new Request.Builder() + .method(method, null) + .url(requestInfo.getURI()) + .addHeader("User-Agent", USER_AGENT); + // Add any additional headers. + requestInfo.getHeaders().forEach(h -> + builder.addHeader(h.getName(), h.getValue())); + extraHeaders.forEach(builder::addHeader); + + if (requestInfo.getUsername() != null && + requestInfo.getSecret() != null) { + builder.addHeader("Authorization", + "Basic " + requestInfo.getBasicAuthToken()); + } + + Request request = builder.build(); + + LOGGER.debug("Requesting {} {} (extra headers: {})", + method, requestInfo.getURI(), extraHeaders); + return getHTTPClient().newCall(request).execute(); } catch (Exception e) { LOGGER.error("request(): {}", e.getMessage()); throw new IOException(e.getMessage(), e); } - - Request.Builder builder = new Request.Builder() - .method(method, null) - .url(requestInfo.getURI()) - .addHeader("User-Agent", getUserAgent()); - // Add any additional headers. - requestInfo.getHeaders().forEach(h -> - builder.addHeader(h.getName(), h.getValue())); - extraHeaders.forEach(builder::addHeader); - - if (requestInfo.getUsername() != null && - requestInfo.getSecret() != null) { - builder.addHeader("Authorization", - "Basic " + requestInfo.getBasicAuthToken()); - } - - Request request = builder.build(); - - LOGGER.debug("Requesting {} {} (extra headers: {})", - method, requestInfo.getURI(), extraHeaders); - - return getHTTPClient().newCall(request).execute(); } /** @@ -540,33 +532,33 @@ HTTPRequestInfo getRequestInfo() throws Exception { if (requestInfo == null) { final LookupStrategy strategy = LookupStrategy.from(Key.HTTPSOURCE_LOOKUP_STRATEGY); - switch (strategy) { - case DELEGATE_SCRIPT: - requestInfo = getRequestInfoUsingScriptStrategy(); - break; - default: - requestInfo = getRequestInfoUsingBasicStrategy(); - break; + if (LookupStrategy.DELEGATE_SCRIPT.equals(strategy)) { + requestInfo = newRequestInfoUsingScriptStrategy(); + } else { + requestInfo = newRequestInfoUsingBasicStrategy(); } } return requestInfo; } - private HTTPRequestInfo getRequestInfoUsingBasicStrategy() { - final Configuration config = Configuration.getInstance(); + private HTTPRequestInfo newRequestInfoUsingBasicStrategy() { + final var config = Configuration.getInstance(); final String prefix = config.getString(Key.HTTPSOURCE_URL_PREFIX, ""); final String suffix = config.getString(Key.HTTPSOURCE_URL_SUFFIX, ""); - return new HTTPRequestInfo( - prefix + identifier.toString() + suffix, - config.getString(Key.HTTPSOURCE_BASIC_AUTH_USERNAME), - config.getString(Key.HTTPSOURCE_BASIC_AUTH_SECRET)); + + final HTTPRequestInfo info = new HTTPRequestInfo(); + info.setURI(prefix + identifier.toString() + suffix); + info.setUsername(config.getString(Key.HTTPSOURCE_BASIC_AUTH_USERNAME)); + info.setSecret(config.getString(Key.HTTPSOURCE_BASIC_AUTH_SECRET)); + info.setSendingHeadRequest(config.getBoolean(Key.HTTPSOURCE_SEND_HEAD_REQUESTS, true)); + return info; } /** * @throws NoSuchFileException if the remote resource was not found. * @throws ScriptException if the delegate method throws an exception. */ - private HTTPRequestInfo getRequestInfoUsingScriptStrategy() + private HTTPRequestInfo newRequestInfoUsingScriptStrategy() throws NoSuchFileException, ScriptException { final DelegateProxy proxy = getDelegateProxy(); final Map result = proxy.getHttpSourceResourceInfo(); @@ -577,13 +569,45 @@ private HTTPRequestInfo getRequestInfoUsingScriptStrategy() " returned nil for " + identifier); } - final String uri = (String) result.get("uri"); - final String username = (String) result.get("username"); - final String secret = (String) result.get("secret"); + final String uri = (String) result.get("uri"); + final String username = (String) result.get("username"); + final String secret = (String) result.get("secret"); @SuppressWarnings("unchecked") - final Map headers = (Map) result.get("headers"); + final Map headers = (Map) result.get("headers"); + final boolean isHeadEnabled = !result.containsKey("send_head_request") || + (boolean) result.get("send_head_request"); + + final HTTPRequestInfo info = new HTTPRequestInfo(); + info.setURI(uri); + info.setUsername(username); + info.setSecret(secret); + info.setHeaders(headers); + info.setSendingHeadRequest(isHeadEnabled); + return info; + } + + @Override + public StreamFactory newStreamFactory() throws IOException { + HTTPRequestInfo info; + try { + info = getRequestInfo(); + } catch (IOException e) { + throw e; + } catch (Exception e) { + LOGGER.error("newStreamFactory(): {}", e.getMessage()); + throw new IOException(e); + } - return new HTTPRequestInfo(uri, username, secret, headers); + if (info != null) { + LOGGER.debug("Resolved {} to {}", identifier, info.getURI()); + getResourceInfo(); + return new HTTPStreamFactory( + getHTTPClient(), + info, + resourceInfo.contentLength(), + resourceInfo.acceptsRanges()); + } + return null; } @Override @@ -593,9 +617,8 @@ public void setIdentifier(Identifier identifier) { } private void reset() { - requestInfo = null; - headResponseInfo = null; - rangedGETResponseInfo = null; + requestInfo = null; + resourceInfo = null; } /** diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/OkHttpHTTPImageInputStreamClient.java b/src/main/java/edu/illinois/library/cantaloupe/source/OkHttpHTTPImageInputStreamClient.java index 4353be05f..d0ac3fc4b 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/OkHttpHTTPImageInputStreamClient.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/OkHttpHTTPImageInputStreamClient.java @@ -53,7 +53,7 @@ public Response sendHEADRequest() throws IOException { Request.Builder builder = new Request.Builder() .method("HEAD", null) .url(uri) - .addHeader("User-Agent", HttpSource.getUserAgent()); + .addHeader("User-Agent", HttpSource.USER_AGENT); extraHeaders.forEach(h -> builder.addHeader(h.getName(), h.getValue())); Request request = builder.build(); @@ -71,7 +71,7 @@ public Response sendGETRequest(Range range) throws IOException { Request.Builder builder = new Request.Builder() .url(uri) .addHeader("Range", "bytes=" + range.start + "-" + range.end) - .addHeader("User-Agent", HttpSource.getUserAgent()); + .addHeader("User-Agent", HttpSource.USER_AGENT); extraHeaders.forEach(h -> builder.addHeader(h.getName(), h.getValue())); Request request = builder.build(); diff --git a/src/main/resources/admin.vm b/src/main/resources/admin.vm index 89e852b9d..7b8a72901 100644 --- a/src/main/resources/admin.vm +++ b/src/main/resources/admin.vm @@ -1721,7 +1721,7 @@ - URL Prefix + Basic Lookup Strategy: URL Prefix - URL Suffix + Basic Lookup Strategy: URL Suffix - Basic Auth Username + Basic Lookup Strategy: Basic Auth Username - Basic Auth Secret + Basic Lookup Strategy: Basic Auth Secret + + + Basic Lookup Strategy: HEAD Requests + ? + + +
      + +
      + + Chunking a[href=\"#JdbcSource\"]").click(); inputNamed(Key.JDBCSOURCE_JDBC_URL).sendKeys("cats://dogs"); @@ -463,6 +464,8 @@ void testSourceSection() throws Exception { config.getString(Key.HTTPSOURCE_BASIC_AUTH_USERNAME)); assertEquals("password", config.getString(Key.HTTPSOURCE_BASIC_AUTH_SECRET)); + assertTrue( + config.getBoolean(Key.HTTPSOURCE_SEND_HEAD_REQUESTS)); // JdbcSource assertEquals("cats://dogs", config.getString(Key.JDBCSOURCE_JDBC_URL)); diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/HTTPRequestInfoTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/HTTPRequestInfoTest.java index 05e93706b..20127c4cc 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/HTTPRequestInfoTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/HTTPRequestInfoTest.java @@ -4,12 +4,11 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.util.HashMap; import java.util.Map; import static org.junit.jupiter.api.Assertions.*; -public class HTTPRequestInfoTest extends BaseTest { +class HTTPRequestInfoTest extends BaseTest { private HTTPRequestInfo instance; @@ -17,22 +16,39 @@ public class HTTPRequestInfoTest extends BaseTest { public void setUp() throws Exception { super.setUp(); - Map headers = new HashMap<>(); - headers.put("X-Animal", "cats"); - - instance = new HTTPRequestInfo("http://example.org/cats", - "user", "secret", headers); + instance = new HTTPRequestInfo(); + instance.setURI("http://example.org/cats"); + instance.setUsername("user"); + instance.setSecret("secret"); + instance.setHeaders(Map.of("X-Animal", "cats")); + instance.setSendingHeadRequest(true); } @Test - void testGetBasicAuthTokenWithoutUserAndSecret() { - instance = new HTTPRequestInfo("http://example.org/cats"); + void getBasicAuthTokenWithoutUserAndSecret() { + instance = new HTTPRequestInfo(); + instance.setURI("http://example.org/cats"); assertNull(instance.getBasicAuthToken()); } @Test - void testGetBasicAuthTokenWithUserAndSecret() { + void getBasicAuthTokenWithUserAndSecret() { assertEquals("dXNlcjpzZWNyZXQ=", instance.getBasicAuthToken()); } + @Test + void setHeaders() { + assertEquals(1, instance.getHeaders().size()); + + instance.setHeaders(Map.of("X-Cats", "yes")); + assertEquals(2, instance.getHeaders().size()); + assertEquals("yes", instance.getHeaders().getFirstValue("X-Cats")); + } + + @Test + void setHeadersWithNullArgument() { + instance.setHeaders(null); + assertEquals(0, instance.getHeaders().size()); + } + } diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/HTTPStreamFactoryTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/HTTPStreamFactoryTest.java index fb570349a..786e6b0d7 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/HTTPStreamFactoryTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/HTTPStreamFactoryTest.java @@ -50,11 +50,11 @@ private HTTPStreamFactory newInstance() { } private HTTPStreamFactory newInstance(boolean serverAcceptsRanges) { - Map headers = new HashMap<>(); - headers.put("X-Custom", "yes"); - HTTPRequestInfo requestInfo = new HTTPRequestInfo( - server.getHTTPURI().resolve("/" + PRESENT_READABLE_IDENTIFIER).toString(), - null, null, headers); + Map headers = Map.of("X-Custom", "yes"); + HTTPRequestInfo requestInfo = new HTTPRequestInfo(); + requestInfo.setURI( + server.getHTTPURI().resolve("/" + PRESENT_READABLE_IDENTIFIER).toString()); + requestInfo.setHeaders(headers); return new HTTPStreamFactory( HttpSource.getHTTPClient(), diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java index fba94db2a..065455c4d 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java @@ -27,12 +27,36 @@ import java.util.HashMap; import java.util.Map; import java.util.NoSuchElementException; -import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.*; abstract class HttpSourceTest extends AbstractSourceTest { + private static class RequestCountingHandler extends DefaultHandler { + + private int numHEADRequests, numGETRequests; + + @Override + public void handle(String target, + Request baseRequest, + HttpServletRequest request, + HttpServletResponse response) { + switch (request.getMethod().toUpperCase()) { + case "HEAD": + numHEADRequests++; + break; + case "GET": + numGETRequests++; + break; + default: + throw new IllegalArgumentException( + "Unexpected method: " + request.getMethod()); + } + baseRequest.setHandled(true); + } + + } + private static final Identifier PRESENT_READABLE_IDENTIFIER = new Identifier("jpg-rgb-64x56x8-baseline.jpg"); @@ -318,15 +342,11 @@ void testCheckAccessWithMalformedURI() throws Exception { config.setProperty(Key.HTTPSOURCE_URL_PREFIX, ""); Identifier identifier = new Identifier( - getServerURI().toString().replace("://", "//") + "/" + PRESENT_READABLE_IDENTIFIER); + getServerURI().toString().replace("://", "//") + "/" + + PRESENT_READABLE_IDENTIFIER); instance.setIdentifier(identifier); - try { - instance.checkAccess(); - fail("Expected exception"); - } catch (IllegalArgumentException e) { - // pass - } + assertThrows(IOException.class, () -> instance.checkAccess()); } /* getFormatIterator() */ @@ -367,10 +387,10 @@ public void handle(String target, server.start(); HttpSource.FormatIterator it = instance.getFormatIterator(); - assertEquals(Format.get("png"), it.next()); // URI path extension - assertEquals(Format.get("png"), it.next()); // identifier extension - assertEquals(Format.UNKNOWN, it.next()); // Content-Type is null - assertEquals(Format.get("jpg"), it.next()); // magic bytes + assertEquals(Format.get("png"), it.next()); // URI path extension + assertEquals(Format.get("png"), it.next()); // identifier extension + assertEquals(Format.UNKNOWN, it.next()); // Content-Type is null + assertEquals(Format.get("jpg"), it.next()); // magic bytes assertThrows(NoSuchElementException.class, it::next); } @@ -478,6 +498,7 @@ void testGetRequestInfoUsingScriptLookupStrategyReturningHash() assertEquals("secret", actual.getSecret()); Headers headers = actual.getHeaders(); assertEquals("yes", headers.getFirstValue("X-Custom")); + assertTrue(actual.isSendingHeadRequest()); } @Test @@ -558,31 +579,39 @@ private void doTestNewStreamFactoryWithPresentReadableImage(Identifier identifie assertNotNull(instance.newStreamFactory()); } + /** + * Simulates a full usage cycle, checking that no unnecessary requests are + * made. + */ @Test - void testNoUnnecessaryRequests() throws Exception { - final AtomicInteger numHEADRequests = new AtomicInteger(0); - final AtomicInteger numGETRequests = new AtomicInteger(0); + void testNoUnnecessaryRequestsWithHEADRequestsEnabled() throws Exception { + final RequestCountingHandler handler = new RequestCountingHandler(); + server.setHandler(handler); + server.start(); - server.setHandler(new DefaultHandler() { - @Override - public void handle(String target, - Request baseRequest, - HttpServletRequest request, - HttpServletResponse response) { - switch (request.getMethod().toUpperCase()) { - case "HEAD": - numHEADRequests.incrementAndGet(); - break; - case "GET": - numGETRequests.incrementAndGet(); - break; - default: - throw new IllegalArgumentException( - "Unexpected method: " + request.getMethod()); - } - baseRequest.setHandled(true); - } - }); + instance.checkAccess(); + instance.getFormatIterator().next(); + + StreamFactory source = instance.newStreamFactory(); + try (InputStream is = source.newInputStream()) { + is.readAllBytes(); + } + + assertEquals(1, handler.numHEADRequests); + assertEquals(1, handler.numGETRequests); + } + + /** + * Simulates a full usage cycle, checking that no unnecessary requests are + * made. + */ + @Test + void testNoUnnecessaryRequestsWithHEADRequestsDisabled() throws Exception { + var config = Configuration.getInstance(); + config.setProperty(Key.HTTPSOURCE_SEND_HEAD_REQUESTS, false); + + final RequestCountingHandler handler = new RequestCountingHandler(); + server.setHandler(handler); server.start(); instance.checkAccess(); @@ -590,12 +619,11 @@ public void handle(String target, StreamFactory source = instance.newStreamFactory(); try (InputStream is = source.newInputStream()) { - //noinspection ResultOfMethodCallIgnored - is.read(); + is.readAllBytes(); } - assertEquals(1, numHEADRequests.get()); - assertEquals(1, numGETRequests.get()); + assertEquals(0, handler.numHEADRequests); + assertEquals(2, handler.numGETRequests); } } diff --git a/src/test/resources/delegates.rb b/src/test/resources/delegates.rb index 7da8b03da..b4d3e8339 100644 --- a/src/test/resources/delegates.rb +++ b/src/test/resources/delegates.rb @@ -181,14 +181,16 @@ def httpsource_resource_info(options = {}) 'uri' => 'http://example.org/bla/' + URI.escape(identifier), 'headers' => { 'X-Custom' => 'yes' - } + }, + 'send_head_request' => true } when 'https-jpg-rgb-64x56x8-baseline.jpg' return { 'uri' => 'https://example.org/bla/' + URI.escape(identifier), 'headers' => { 'X-Custom' => 'yes' - } + }, + 'send_head_request' => true } when 'http-jpg-rgb-64x56x8-plane.jpg' return { @@ -197,7 +199,8 @@ def httpsource_resource_info(options = {}) 'secret' => 'secret', 'headers' => { 'X-Custom' => 'yes' - } + }, + 'send_head_request' => true } when 'https-jpg-rgb-64x56x8-plane.jpg' return { @@ -206,7 +209,8 @@ def httpsource_resource_info(options = {}) 'secret' => 'secret', 'headers' => { 'X-Custom' => 'yes' - } + }, + 'send_head_request' => true } end nil From 78c95c733c37a2dbd342136f8a61bcfd1986e611 Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Fri, 28 May 2021 14:34:09 -0500 Subject: [PATCH 004/106] Set version to 6.0-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 20c94d51c..cb7236578 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ edu.illinois.library.cantaloupe cantaloupe jar - 5.0.3-SNAPSHOT + 6.0-SNAPSHOT Cantaloupe https://cantaloupe-project.github.io/ 2015 From 3cd5e332d5c72ee28c9f53ada0b96eaecdacb45b Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Wed, 2 Jun 2021 14:49:14 -0500 Subject: [PATCH 005/106] Add DerivativeCache.purgeInfos() --- .../cantaloupe/cache/AzureStorageCache.java | 30 +++++++++- .../cantaloupe/cache/DerivativeCache.java | 10 ++++ .../cantaloupe/cache/FilesystemCache.java | 40 +++++++++++++- .../library/cantaloupe/cache/HeapCache.java | 7 ++- .../cantaloupe/cache/InfoFileVisitor.java | 55 +++++++++++++++++++ .../library/cantaloupe/cache/JdbcCache.java | 17 ++++++ .../library/cantaloupe/cache/RedisCache.java | 32 +++++++---- .../library/cantaloupe/cache/S3Cache.java | 32 +++++++++-- .../cantaloupe/cache/AbstractCacheTest.java | 51 ++++++++++++++++- .../cache/MockBrokenDerivativeCache.java | 5 ++ .../library/cantaloupe/cache/MockCache.java | 11 +++- 11 files changed, 267 insertions(+), 23 deletions(-) create mode 100644 src/main/java/edu/illinois/library/cantaloupe/cache/InfoFileVisitor.java diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java index 28e992cc5..573ae1cc4 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java @@ -143,6 +143,8 @@ public void write(byte[] b, int off, int len) throws IOException { private static final Logger LOGGER = LoggerFactory.getLogger(AzureStorageCache.class); + private static final String INFO_EXTENSION = ".json"; + private static CloudBlobClient client; /** @@ -289,7 +291,7 @@ public InputStream newDerivativeImageInputStream(OperationList opList) */ String getObjectKey(Identifier identifier) { return getObjectKeyPrefix() + "info/" + - StringUtils.md5(identifier.toString()) + ".json"; + StringUtils.md5(identifier.toString()) + INFO_EXTENSION; } /** @@ -389,6 +391,32 @@ private void purgeAsync(CloudBlob blob) { }); } + @Override + public void purgeInfos() throws IOException { + final String containerName = getContainerName(); + final CloudBlobClient client = getClientInstance(); + try { + final CloudBlobContainer container = + client.getContainerReference(containerName); + int count = 0, deletedCount = 0; + for (ListBlobItem item : container.listBlobs(getObjectKeyPrefix(), true)) { + if (item instanceof CloudBlob) { + CloudBlob blob = (CloudBlob) item; + count++; + if (blob.getName().endsWith(INFO_EXTENSION)) { + if (blob.deleteIfExists()) { + deletedCount++; + } + } + } + } + LOGGER.debug("purgeInfos(): deleted {} of {} items", + deletedCount, count); + } catch (URISyntaxException | StorageException e) { + throw new IOException(e.getMessage(), e); + } + } + @Override public void purgeInvalid() throws IOException { final String containerName = getContainerName(); diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java index 31d1aeb64..3827d591e 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java @@ -80,6 +80,16 @@ CompletableOutputStream newDerivativeImageOutputStream(OperationList opList) */ void purge(OperationList opList) throws IOException; + /** + * Deletes all cached infos. + * + * @throws IOException upon fatal error. Implementations should do the + * best they can to complete the operation and swallow and log + * non-fatal errors. + * @since 5.0 + */ + void purgeInfos() throws IOException; + /** *

      Synchronously adds image information to the cache.

      * diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java index 92596e84a..8990ba487 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java @@ -238,7 +238,7 @@ public void write(byte[] b, int off, int len) throws IOException { private static final String INFO_FOLDER = "info"; private static final String SOURCE_IMAGE_FOLDER = "source"; - private static final String INFO_EXTENSION = ".json"; + static final String INFO_EXTENSION = ".json"; private static final String TEMP_EXTENSION = ".tmp"; /** @@ -846,6 +846,44 @@ private void purgeAsync(final Path path) { }); } + @Override + public void purgeInfos() throws IOException { + if (isGlobalPurgeInProgress.get()) { + LOGGER.debug("purgeInfos() called with a purge in progress. Aborting."); + return; + } + synchronized (infoPurgeLock) { + while (!infosBeingPurged.isEmpty()) { + try { + LOGGER.debug("purgeInfos(): waiting..."); + infoPurgeLock.wait(); + } catch (InterruptedException e) { + break; + } + } + } + + try { + isGlobalPurgeInProgress.set(true); + + final InfoFileVisitor visitor = new InfoFileVisitor(); + + LOGGER.debug("purgeInfos(): starting..."); + Files.walkFileTree(rootPath(), + EnumSet.of(FileVisitOption.FOLLOW_LINKS), + Integer.MAX_VALUE, + visitor); + LOGGER.debug("purgeInfos(): purged {} info(s) totaling {} bytes", + visitor.getDeletedFileCount(), + visitor.getDeletedFileSize()); + } finally { + isGlobalPurgeInProgress.set(false); + synchronized (infoPurgeLock) { + infoPurgeLock.notifyAll(); + } + } + } + /** *

      Crawls the image directory, deleting all expired files within it * (temporary or not), and then does the same in the info directory.

      diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/HeapCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/HeapCache.java index 1cc7c0d0d..a7e572a1c 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/HeapCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/HeapCache.java @@ -233,7 +233,7 @@ public void run() { if (workerShouldWork.get()) { try { purgeExcess(); - logger.debug("Cache size: {} items ({} bytes)", + logger.trace("Cache size: {} items ({} bytes)", size(), getByteSize()); Thread.sleep(INTERVAL_SECONDS * 1000); } catch (ConfigurationException e) { @@ -590,6 +590,11 @@ void purgeExcess() throws ConfigurationException { } } + @Override + public void purgeInfos() { + cache.entrySet().removeIf(entry -> entry.getKey().opList == null); + } + /** * Does nothing, as items in this cache never expire. */ diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/InfoFileVisitor.java b/src/main/java/edu/illinois/library/cantaloupe/cache/InfoFileVisitor.java new file mode 100644 index 000000000..d8d91f80f --- /dev/null +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/InfoFileVisitor.java @@ -0,0 +1,55 @@ +package edu.illinois.library.cantaloupe.cache; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; + +/** + * Used by {@link Files#walkFileTree} to delete all info files within a + * directory. + */ +class InfoFileVisitor extends SimpleFileVisitor { + + private static final Logger LOGGER = + LoggerFactory.getLogger(InfoFileVisitor.class); + + private long deletedFileCount = 0; + private long deletedFileSize = 0; + + long getDeletedFileCount() { + return deletedFileCount; + } + + long getDeletedFileSize() { + return deletedFileSize; + } + + @Override + public FileVisitResult visitFile(Path path, + BasicFileAttributes attrs) { + try { + if (path.toString().endsWith(FilesystemCache.INFO_EXTENSION)) { + long size = Files.size(path); + Files.delete(path); + deletedFileCount++; + deletedFileSize += size; + } + } catch (IOException e) { + LOGGER.warn(e.getMessage(), e); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e) { + LOGGER.warn("visitFileFailed(): {}", e.getMessage()); + return FileVisitResult.CONTINUE; + } + +} diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java index 8c2d4f59e..f68506cc6 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java @@ -427,6 +427,23 @@ public void purge(OperationList ops) throws IOException { } } + @Override + public void purgeInfos() throws IOException { + try (Connection connection = getConnection()) { + connection.setAutoCommit(false); + int numDeleted; + final String sql = "DELETE FROM " + getInfoTableName(); + try (PreparedStatement statement = connection.prepareStatement(sql)) { + LOGGER.trace(sql); + numDeleted = statement.executeUpdate(); + } + connection.commit(); + LOGGER.debug("purgeInfos(): purged {} info(s)", numDeleted); + } catch (SQLException e) { + throw new IOException(e.getMessage(), e); + } + } + @Override public void purgeInvalid() throws IOException { try (Connection connection = getConnection()) { diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/RedisCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/RedisCache.java index 60d76cba5..e7743aea0 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/RedisCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/RedisCache.java @@ -31,14 +31,16 @@ * *

      Content is structured as follows:

      * - * 
      {
+ * {@code {
  *     #{@link #IMAGE_HASH_KEY}: {
  *         "operation list string representation": image byte array
  *     },
  *     #{@link #INFO_HASH_KEY}: {
  *         "identifier": "UTF-8 JSON string"
  *     }
- * }
       + * }} + * + * @since 3.4 */ class RedisCache implements DerivativeCache { @@ -202,7 +204,7 @@ private static synchronized StatefulRedisConnection getConnectio RedisURI.Builder.redis(config.getString(Key.REDISCACHE_HOST)). withPort(config.getInt(Key.REDISCACHE_PORT, 6379)). withSsl(config.getBoolean(Key.REDISCACHE_SSL, false)). - withPassword(config.getString(Key.REDISCACHE_PASSWORD, "")). + withPassword(config.getString(Key.REDISCACHE_PASSWORD, "").toCharArray()). withDatabase(config.getInt(Key.REDISCACHE_DATABASE, 0)). build(); RedisClient client = RedisClient.create(redisUri); @@ -249,13 +251,8 @@ public InputStream newDerivativeImageInputStream(OperationList opList) { @Override public void purge() { - // Purge infos - LOGGER.debug("purge(): purging {}...", INFO_HASH_KEY); - getConnection().sync().del(INFO_HASH_KEY); - - // Purge images - LOGGER.debug("purge(): purging {}...", IMAGE_HASH_KEY); - getConnection().sync().del(IMAGE_HASH_KEY); + purgeInfos(); + purgeImages(); } @Override @@ -271,11 +268,22 @@ public void purge(Identifier identifier) { MapScanCursor cursor = getConnection().sync(). hscan(IMAGE_HASH_KEY, imagePattern); - for (Object key : cursor.getMap().keySet()) { - getConnection().sync().hdel(IMAGE_HASH_KEY, (String) key); + for (String key : cursor.getMap().keySet()) { + getConnection().sync().hdel(IMAGE_HASH_KEY, key); } } + private void purgeImages() { + LOGGER.debug("purgeImages(): purging {}...", IMAGE_HASH_KEY); + getConnection().sync().del(IMAGE_HASH_KEY); + } + + @Override + public void purgeInfos() { + LOGGER.debug("purgeInfos(): purging {}...", INFO_HASH_KEY); + getConnection().sync().del(INFO_HASH_KEY); + } + /** * No-op. */ diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java index 5c977fc67..a86ccf67e 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java @@ -216,6 +216,10 @@ public void run() { private static final Logger LOGGER = LoggerFactory.getLogger(S3Cache.class); + private static final String IMAGE_KEY_PREFIX = "image/"; + private static final String INFO_EXTENSION = ".json"; + private static final String INFO_KEY_PREFIX = "info/"; + /** * Lazy-initialized by {@link #getClientInstance}. */ @@ -352,8 +356,8 @@ public InputStream newDerivativeImageInputStream(OperationList opList) * given identifier. */ String getObjectKey(Identifier identifier) { - return getObjectKeyPrefix() + "info/" + - StringUtils.md5(identifier.toString()) + ".json"; + return getObjectKeyPrefix() + INFO_KEY_PREFIX + + StringUtils.md5(identifier.toString()) + INFO_EXTENSION; } /** @@ -369,8 +373,8 @@ String getObjectKey(OperationList opList) { if (encode != null) { extension = "." + encode.getFormat().getPreferredExtension(); } - return String.format("%simage/%s/%s%s", - getObjectKeyPrefix(), idHash, opsHash, extension); + return getObjectKeyPrefix() + IMAGE_KEY_PREFIX + idHash + "/" + + opsHash + extension; } /** @@ -414,7 +418,7 @@ public void purge(final Identifier identifier) { // purge images final S3Client client = getClientInstance(); final String bucketName = getBucketName(); - final String prefix = getObjectKeyPrefix() + "image/" + + final String prefix = getObjectKeyPrefix() + IMAGE_KEY_PREFIX + StringUtils.md5(identifier.toString()); final AtomicInteger counter = new AtomicInteger(); @@ -455,6 +459,24 @@ private void purgeAsync(final String bucketName, final String key) { }); } + @Override + public void purgeInfos() { + final S3Client client = getClientInstance(); + final String bucketName = getBucketName(); + final String prefix = getObjectKeyPrefix() + INFO_KEY_PREFIX; + final AtomicInteger counter = new AtomicInteger(); + + S3Utils.walkObjects(client, bucketName, prefix, (object) -> { + LOGGER.trace("purgeInfos(): deleting {}", object.key()); + client.deleteObject(DeleteObjectRequest.builder() + .bucket(bucketName) + .key(object.key()) + .build()); + counter.incrementAndGet(); + }); + LOGGER.debug("purgeInfos(): deleted {} items", counter.get()); + } + @Override public void purgeInvalid() { final S3Client client = getClientInstance(); diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java index 240ab4960..2cfc19718 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java @@ -411,13 +411,60 @@ void testPurgeWithOperationList() throws Exception { assertExists(instance, ops2); } + /* purgeInfos() */ + + @Test + void testPurgeInfos() throws Exception { + DerivativeCache instance = newInstance(); + Identifier identifier = new Identifier(IMAGE); + OperationList opList = OperationList.builder() + .withIdentifier(identifier) + .withOperations(new Encode(Format.get("jpg"))) + .build(); + Info info = new Info(); + + // assert that a particular image doesn't exist + try (InputStream is = instance.newDerivativeImageInputStream(opList)) { + assertNull(is); + } + + // assert that a particular info doesn't exist + assertFalse(instance.getInfo(identifier).isPresent()); + + // add the image + try (CompletableOutputStream outputStream = + instance.newDerivativeImageOutputStream(opList)) { + Path fixture = TestUtil.getImage(IMAGE); + Files.copy(fixture, outputStream); + outputStream.setCompletelyWritten(true); + } + + // add the info + instance.put(identifier, info); + + Thread.sleep(ASYNC_WAIT); + + // assert that they've been added + assertExists(instance, opList); + assertNotNull(instance.getInfo(identifier)); + + // purge infos + instance.purgeInfos(); + + // assert that the info has been purged + assertFalse(instance.getInfo(identifier).isPresent()); + + // assert that the image has NOT been purged + assertExists(instance, opList); + } + /* purgeInvalid() */ @Test void testPurgeInvalid() throws Exception { DerivativeCache instance = newInstance(); - Identifier id1 = new Identifier(IMAGE); - OperationList ops1 = OperationList.builder() + Identifier id1 = new Identifier(IMAGE); + OperationList ops1 = OperationList.builder() .withIdentifier(id1) .withOperations(new Encode(Format.get("jpg"))) .build(); diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/MockBrokenDerivativeCache.java b/src/test/java/edu/illinois/library/cantaloupe/cache/MockBrokenDerivativeCache.java index 9d935090e..8bbc22f5e 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/MockBrokenDerivativeCache.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/MockBrokenDerivativeCache.java @@ -41,6 +41,11 @@ public void purge(OperationList opList) throws IOException { throw new IOException("I'm broken"); } + @Override + public void purgeInfos() throws IOException { + throw new IOException("I'm broken"); + } + @Override public void purgeInvalid() throws IOException { throw new IOException("I'm broken"); diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/MockCache.java b/src/test/java/edu/illinois/library/cantaloupe/cache/MockCache.java index 798f0a44a..e742404e7 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/MockCache.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/MockCache.java @@ -13,7 +13,7 @@ class MockCache implements DerivativeCache, SourceCache { private boolean isCleanUpCalled, isInitializeCalled, isOnCacheWorkerCalled, - isPurgeInvalidCalled, isShutdownCalled; + isPurgeInfosCalled, isPurgeInvalidCalled, isShutdownCalled; @Override public void cleanUp() { @@ -48,6 +48,10 @@ boolean isOnCacheWorkerCalled() { return isOnCacheWorkerCalled; } + public boolean isPurgeInfosCalled() { + return isPurgeInfosCalled; + } + boolean isPurgeInvalidCalled() { return isPurgeInvalidCalled; } @@ -89,6 +93,11 @@ public void purge(Identifier identifier) {} @Override public void purge(OperationList opList) {} + @Override + public void purgeInfos() { + isPurgeInfosCalled = true; + } + @Override public void purgeInvalid() { isPurgeInvalidCalled = true; From fe4dc999e607602958d2ffc143c31775433a5a4f Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Wed, 2 Jun 2021 14:49:29 -0500 Subject: [PATCH 006/106] Log SQL at trace level --- .../library/cantaloupe/cache/JdbcCache.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java index f68506cc6..95a6a57de 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java @@ -78,7 +78,7 @@ private class ImageBlobOutputStream extends CompletableOutputStream { DERIVATIVE_IMAGE_TABLE_OPERATIONS_COLUMN, DERIVATIVE_IMAGE_TABLE_IMAGE_COLUMN, DERIVATIVE_IMAGE_TABLE_LAST_ACCESSED_COLUMN); - LOGGER.debug(sql); + LOGGER.trace(sql); final Blob blob = connection.createBlob(); blobOutputStream = blob.setBinaryStream(1); @@ -250,7 +250,7 @@ private void accessDerivativeImage(OperationList opList, statement.setTimestamp(1, now()); statement.setString(2, opList.toString()); - LOGGER.debug(sql); + LOGGER.trace(sql); statement.executeUpdate(); } } @@ -285,7 +285,7 @@ private void accessInfo(Identifier identifier, Connection connection) statement.setTimestamp(1, now()); statement.setString(2, identifier.toString()); - LOGGER.debug(sql); + LOGGER.trace(sql); statement.executeUpdate(); } } @@ -329,7 +329,7 @@ public Optional getInfo(Identifier identifier) throws IOException { statement.setString(1, identifier.toString()); statement.setTimestamp(2, earliestValidDate()); - LOGGER.debug(sql); + LOGGER.trace(sql); try (ResultSet resultSet = statement.executeQuery()) { if (resultSet.next()) { accessInfoAsync(identifier); @@ -365,7 +365,7 @@ public InputStream newDerivativeImageInputStream(OperationList opList) statement.setString(1, opList.toString()); statement.setTimestamp(2, earliestValidDate()); - LOGGER.debug(sql); + LOGGER.trace(sql); try (ResultSet resultSet = statement.executeQuery()) { if (resultSet.next()) { LOGGER.debug("Hit for image: {}", opList); @@ -470,7 +470,7 @@ private int purgeExpiredDerivativeImages(Connection conn) DERIVATIVE_IMAGE_TABLE_LAST_ACCESSED_COLUMN); try (PreparedStatement statement = conn.prepareStatement(sql)) { statement.setTimestamp(1, earliestValidDate()); - LOGGER.debug(sql); + LOGGER.trace(sql); return statement.executeUpdate(); } } @@ -485,7 +485,7 @@ private int purgeExpiredInfos(Connection conn) getInfoTableName(), INFO_TABLE_LAST_ACCESSED_COLUMN); try (PreparedStatement statement = conn.prepareStatement(sql)) { statement.setTimestamp(1, earliestValidDate()); - LOGGER.debug(sql); + LOGGER.trace(sql); return statement.executeUpdate(); } } @@ -502,7 +502,7 @@ private int purgeDerivativeImage(OperationList ops, Connection conn) DERIVATIVE_IMAGE_TABLE_OPERATIONS_COLUMN); try (PreparedStatement statement = conn.prepareStatement(sql)) { statement.setString(1, ops.toString()); - LOGGER.debug(sql); + LOGGER.trace(sql); return statement.executeUpdate(); } } @@ -529,7 +529,7 @@ private void purgeDerivativeImageAsync(OperationList ops) { private int purgeDerivativeImages(Connection conn) throws SQLException { final String sql = "DELETE FROM " + getDerivativeImageTableName(); try (PreparedStatement statement = conn.prepareStatement(sql)) { - LOGGER.debug(sql); + LOGGER.trace(sql); return statement.executeUpdate(); } } @@ -549,7 +549,7 @@ private int purgeDerivativeImages(Identifier identifier, Connection conn) " LIKE ?"; try (PreparedStatement statement = conn.prepareStatement(sql)) { statement.setString(1, identifier.toString() + "%"); - LOGGER.debug(sql); + LOGGER.trace(sql); return statement.executeUpdate(); } } @@ -583,7 +583,7 @@ private int purgeInfo(Identifier identifier, Connection conn) getInfoTableName(), INFO_TABLE_IDENTIFIER_COLUMN); try (PreparedStatement statement = conn.prepareStatement(sql)) { statement.setString(1, identifier.toString()); - LOGGER.debug(sql); + LOGGER.trace(sql); return statement.executeUpdate(); } } @@ -605,7 +605,7 @@ private void purgeInfoAsync(Identifier identifier) { private int purgeInfos(Connection conn) throws SQLException { final String sql = "DELETE FROM " + getInfoTableName(); try (PreparedStatement statement = conn.prepareStatement(sql)) { - LOGGER.debug(sql); + LOGGER.trace(sql); return statement.executeUpdate(); } } @@ -613,7 +613,7 @@ private int purgeInfos(Connection conn) throws SQLException { @Override public void put(Identifier identifier, Info info) throws IOException { if (!info.isPersistable()) { - LOGGER.debug("put(): info for {} is incomplete; ignoring", + LOGGER.trace("put(): info for {} is incomplete; ignoring", identifier); return; } From a740205d47599b0df4d99df66be2c35c0d68bd4f Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Wed, 2 Jun 2021 15:02:47 -0500 Subject: [PATCH 007/106] Add PurgeInfosFromCacheCommand --- CHANGES.md | 1 + .../library/cantaloupe/cache/CacheFacade.java | 11 ++++++++ .../cantaloupe/resource/api/Command.java | 3 ++ .../api/PurgeInfosFromCacheCommand.java | 24 ++++++++++++++++ .../cantaloupe/cache/CacheFacadeTest.java | 28 +++++++++++++++++++ 5 files changed, 67 insertions(+) create mode 100644 src/main/java/edu/illinois/library/cantaloupe/resource/api/PurgeInfosFromCacheCommand.java diff --git a/CHANGES.md b/CHANGES.md index 79d0b2c73..8d7d97592 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ * HttpSource can be configured to send a ranged GET request instead of a HEAD request, enabling it to work with pre-signed URLs that do not allow HEAD requests. +* Added an HTTP API method to purge all infos from the derivative cache. * The delegate script pathname can be set using the `-Dcantaloupe.delegate_script` VM argument, which takes precedence over the `delegate_script.pathname` configuration key. diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/CacheFacade.java b/src/main/java/edu/illinois/library/cantaloupe/cache/CacheFacade.java index 78ff159e5..9bdbfcf92 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/CacheFacade.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/CacheFacade.java @@ -186,6 +186,17 @@ public void purge(OperationList opList) throws IOException { } } + /** + * @see DerivativeCache#purgeInfos() + * @since 6.0 + */ + public void purgeInfos() throws IOException { + Optional optCache = getDerivativeCache(); + if (optCache.isPresent()) { + optCache.get().purgeInfos(); + } + } + /** * @see Cache#purgeInvalid */ diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/api/Command.java b/src/main/java/edu/illinois/library/cantaloupe/resource/api/Command.java index a7d9b2e8d..5722814e0 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/api/Command.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/api/Command.java @@ -24,6 +24,9 @@ @JsonSubTypes.Type( name = "PurgeInfoCache", value = PurgeInfoCacheCommand.class), + @JsonSubTypes.Type( + name = "PurgeInfosFromCache", + value = PurgeInfosFromCacheCommand.class), @JsonSubTypes.Type( name = "PurgeInvalidFromCache", value = PurgeInvalidFromCacheCommand.class), diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/api/PurgeInfosFromCacheCommand.java b/src/main/java/edu/illinois/library/cantaloupe/resource/api/PurgeInfosFromCacheCommand.java new file mode 100644 index 000000000..d0cbbb9c4 --- /dev/null +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/api/PurgeInfosFromCacheCommand.java @@ -0,0 +1,24 @@ +package edu.illinois.library.cantaloupe.resource.api; + +import edu.illinois.library.cantaloupe.cache.CacheFacade; + +import java.util.concurrent.Callable; + +/** + * @since 6.0 + */ +final class PurgeInfosFromCacheCommand extends Command + implements Callable { + + @Override + public T call() throws Exception { + new CacheFacade().purgeInfos(); + return null; + } + + @Override + String getVerb() { + return "PurgeInfosFromCache"; + } + +} diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/CacheFacadeTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/CacheFacadeTest.java index 13c490d59..d1350362e 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/CacheFacadeTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/CacheFacadeTest.java @@ -392,6 +392,34 @@ void testPurgeWithOperationList() throws Exception { } } + /* purgeInfos() */ + + @Test + void testPurgeInfos() throws Exception { + final Configuration config = Configuration.getInstance(); + config.setProperty(Key.DERIVATIVE_CACHE_TTL, 1); + + enableDerivativeCache(); + DerivativeCache derivCache = CacheFactory.getDerivativeCache().get(); + + Identifier identifier = new Identifier("jpg"); + Info info = new Info(); + + // Add info to the derivative cache. + derivCache.put(identifier, info); + + // Assert that everything has been added. + assertNotNull(derivCache.getInfo(identifier)); + + instance.purgeInfos(); + + Thread.sleep(ASYNC_WAIT); + + // Assert that it's gone. + assertEquals(0, InfoService.getInstance().getInfoCache().size()); + assertFalse(derivCache.getInfo(identifier).isPresent()); + } + /* purgeInvalid() */ @Test From dd235ba54a0205ac9bd67ab0c015b40c5ae29f5b Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Tue, 29 Jun 2021 17:43:26 -0500 Subject: [PATCH 008/106] Add endpoint.health.enabled and repackage HealthResource --- CHANGES.md | 2 ++ UPGRADING.md | 1 + cantaloupe.properties.sample | 3 +++ .../library/cantaloupe/config/Key.java | 1 + .../library/cantaloupe/resource/Route.java | 2 +- .../{api => health}/HealthResource.java | 22 +++++++++++----- src/main/resources/admin.vm | 19 ++++++++++++++ .../cantaloupe/resource/RouteTest.java | 4 +-- .../resource/admin/AdminResourceUITest.java | 4 +++ .../{api => health}/HealthResourceTest.java | 26 +++++-------------- 10 files changed, 55 insertions(+), 29 deletions(-) rename src/main/java/edu/illinois/library/cantaloupe/resource/{api => health}/HealthResource.java (75%) rename src/test/java/edu/illinois/library/cantaloupe/resource/{api => health}/HealthResourceTest.java (88%) diff --git a/CHANGES.md b/CHANGES.md index 60f0189ad..fc78b6457 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ # 6.0 +* The health endpoint is en/disabled via `endpoint.health.enabled` rather than + `endpoint.api.enabled`. * HttpSource can be configured to send a ranged GET request instead of a HEAD request, enabling it to work with pre-signed URLs that do not allow HEAD requests. diff --git a/UPGRADING.md b/UPGRADING.md index 65a474b7e..587efd51c 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -6,6 +6,7 @@ current version. ## 5.0 → 6.0 1. Add the following keys from the sample configuration: + * `endpoint.health.enabled` * `HttpSource.BasicLookupStrategy.send_head_requests` ## 4.1.x → 5.0 diff --git a/cantaloupe.properties.sample b/cantaloupe.properties.sample index 3d6248650..e08bdd414 100644 --- a/cantaloupe.properties.sample +++ b/cantaloupe.properties.sample @@ -128,6 +128,9 @@ endpoint.api.enabled = false endpoint.api.username = endpoint.api.secret = +# Enables the health check endpoint, at /health. +endpoint.health.enabled = true + # If true, sources and caches will be checked, resulting in a more robust # but slower health check. Set this to false if these services already have # their own health checks. diff --git a/src/main/java/edu/illinois/library/cantaloupe/config/Key.java b/src/main/java/edu/illinois/library/cantaloupe/config/Key.java index 3f53ff0af..2de3667ef 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/config/Key.java +++ b/src/main/java/edu/illinois/library/cantaloupe/config/Key.java @@ -91,6 +91,7 @@ public enum Key { FILESYSTEMSOURCE_PATH_PREFIX("FilesystemSource.BasicLookupStrategy.path_prefix"), FILESYSTEMSOURCE_PATH_SUFFIX("FilesystemSource.BasicLookupStrategy.path_suffix"), HEALTH_DEPENDENCY_CHECK("endpoint.health.dependency_check"), + HEALTH_ENDPOINT_ENABLED("endpoint.health.enabled"), HEAPCACHE_PATHNAME("HeapCache.persist.filesystem.pathname"), HEAPCACHE_PERSIST("HeapCache.persist"), HEAPCACHE_TARGET_SIZE("HeapCache.target_size"), diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/Route.java b/src/main/java/edu/illinois/library/cantaloupe/resource/Route.java index 1b4fe0d45..4bfb4111e 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/Route.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/Route.java @@ -89,7 +89,7 @@ public final class Route { MAPPINGS.put(Pattern.compile("^" + CONFIGURATION_PATH + "$"), edu.illinois.library.cantaloupe.resource.api.ConfigurationResource.class); MAPPINGS.put(Pattern.compile("^" + HEALTH_PATH + "$"), - edu.illinois.library.cantaloupe.resource.api.HealthResource.class); + edu.illinois.library.cantaloupe.resource.health.HealthResource.class); MAPPINGS.put(Pattern.compile("^" + STATUS_PATH + "$"), edu.illinois.library.cantaloupe.resource.api.StatusResource.class); MAPPINGS.put(Pattern.compile("^" + TASKS_PATH + "$"), diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/api/HealthResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/health/HealthResource.java similarity index 75% rename from src/main/java/edu/illinois/library/cantaloupe/resource/api/HealthResource.java rename to src/main/java/edu/illinois/library/cantaloupe/resource/health/HealthResource.java index 89d323dc2..854998c9f 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/api/HealthResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/health/HealthResource.java @@ -1,9 +1,11 @@ -package edu.illinois.library.cantaloupe.resource.api; +package edu.illinois.library.cantaloupe.resource.health; import com.fasterxml.jackson.databind.SerializationFeature; import edu.illinois.library.cantaloupe.config.Configuration; import edu.illinois.library.cantaloupe.config.Key; import edu.illinois.library.cantaloupe.http.Method; +import edu.illinois.library.cantaloupe.resource.AbstractResource; +import edu.illinois.library.cantaloupe.resource.EndpointDisabledException; import edu.illinois.library.cantaloupe.resource.JacksonRepresentation; import edu.illinois.library.cantaloupe.status.Health; import edu.illinois.library.cantaloupe.status.HealthChecker; @@ -16,7 +18,7 @@ /** * Provides health checks via the HTTP API. */ -public class HealthResource extends AbstractAPIResource { +public class HealthResource extends AbstractResource { private static final Logger LOGGER = LoggerFactory.getLogger(HealthResource.class); @@ -27,6 +29,17 @@ public class HealthResource extends AbstractAPIResource { private static final Map SERIALIZATION_FEATURES = Map.of(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true); + @Override + public void doInit() throws Exception { + super.doInit(); + getResponse().setHeader("Cache-Control", "no-cache"); + + final Configuration config = Configuration.getInstance(); + if (!config.getBoolean(Key.HEALTH_ENDPOINT_ENABLED, false)) { + throw new EndpointDisabledException(); + } + } + @Override protected Logger getLogger() { return LOGGER; @@ -57,9 +70,4 @@ public void doGET() throws IOException { SERIALIZATION_FEATURES); } - @Override - boolean requiresAuth() { - return false; - } - } diff --git a/src/main/resources/admin.vm b/src/main/resources/admin.vm index 7b8a72901..2134b10dc 100644 --- a/src/main/resources/admin.vm +++ b/src/main/resources/admin.vm @@ -1121,6 +1121,25 @@

      Health Check

      + + + + + + + + + + + + + + + +
      + ? + +
      + +
      +
      a[href=\"#StandardMetaIdentifierTransformer\"]").click(); inputNamed(Key.STANDARD_META_IDENTIFIER_TRANSFORMER_DELIMITER).sendKeys("---"); @@ -313,6 +315,8 @@ void testEndpointsSection() throws Exception { assertTrue(config.getBoolean(Key.API_ENABLED)); assertEquals("cats", config.getString(Key.API_USERNAME)); assertEquals("dogs", config.getString(Key.API_SECRET)); + assertTrue(config.getBoolean(Key.HEALTH_ENDPOINT_ENABLED)); + assertTrue(config.getBoolean(Key.HEALTH_DEPENDENCY_CHECK)); assertEquals("StandardMetaIdentifierTransformer", config.getString(Key.META_IDENTIFIER_TRANSFORMER)); assertEquals("---", diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/api/HealthResourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/health/HealthResourceTest.java similarity index 88% rename from src/test/java/edu/illinois/library/cantaloupe/resource/api/HealthResourceTest.java rename to src/test/java/edu/illinois/library/cantaloupe/resource/health/HealthResourceTest.java index 0d029493f..97a3a02e8 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/api/HealthResourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/health/HealthResourceTest.java @@ -1,4 +1,4 @@ -package edu.illinois.library.cantaloupe.resource.api; +package edu.illinois.library.cantaloupe.resource.health; import edu.illinois.library.cantaloupe.Application; import edu.illinois.library.cantaloupe.config.Configuration; @@ -7,6 +7,7 @@ import edu.illinois.library.cantaloupe.http.Headers; import edu.illinois.library.cantaloupe.http.ResourceException; import edu.illinois.library.cantaloupe.http.Response; +import edu.illinois.library.cantaloupe.resource.ResourceTest; import edu.illinois.library.cantaloupe.resource.Route; import edu.illinois.library.cantaloupe.status.Health; import edu.illinois.library.cantaloupe.status.HealthChecker; @@ -17,7 +18,7 @@ import static org.junit.jupiter.api.Assertions.*; -public class HealthResourceTest extends AbstractAPIResourceTest { +public class HealthResourceTest extends ResourceTest { @BeforeEach @Override @@ -25,6 +26,9 @@ public void setUp() throws Exception { super.setUp(); HealthChecker.getSourceUsages().clear(); HealthChecker.overrideHealth(null); + Configuration config = Configuration.getInstance(); + config.setProperty(Key.HEALTH_ENDPOINT_ENABLED, true); + client = newClient(""); } @Override @@ -35,7 +39,7 @@ protected String getEndpointPath() { @Test void testGETWithEndpointDisabled() throws Exception { Configuration config = Configuration.getInstance(); - config.setProperty(Key.API_ENABLED, false); + config.setProperty(Key.HEALTH_ENDPOINT_ENABLED, false); try { client.send(); fail("Expected exception"); @@ -116,22 +120,6 @@ void testGETWithRedStatus() throws Exception { } } - @Override // because this endpoint doesn't require auth - @Test - public void testGETWithNoCredentials() throws Exception { - client.setUsername(null); - client.setSecret(null); - client.send(); - } - - @Override // because this endpoint doesn't require auth - @Test - public void testGETWithInvalidCredentials() throws Exception { - client.setUsername("invalid"); - client.setSecret("invalid"); - client.send(); - } - @Test void testGETResponseBody() throws Exception { Response response = client.send(); From 4b79857f4ea2fcaa42c0cf0d4343111103641df2 Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Tue, 29 Jun 2021 17:43:45 -0500 Subject: [PATCH 009/106] Remove requiresAuth() --- .../resource/api/AbstractAPIResource.java | 18 +++++++----------- .../resource/api/ConfigurationResource.java | 5 ----- .../resource/api/StatusResource.java | 5 ----- .../cantaloupe/resource/api/TaskResource.java | 5 ----- .../cantaloupe/resource/api/TasksResource.java | 5 ----- 5 files changed, 7 insertions(+), 31 deletions(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/api/AbstractAPIResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/api/AbstractAPIResource.java index 6c5796ace..e41473d71 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/api/AbstractAPIResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/api/AbstractAPIResource.java @@ -22,17 +22,13 @@ public void doInit() throws Exception { throw new EndpointDisabledException(); } - if (requiresAuth()) { - authenticateUsingBasic(BASIC_REALM, user -> { - final String configUser = config.getString(Key.API_USERNAME, ""); - if (!configUser.isEmpty() && configUser.equals(user)) { - return config.getString(Key.API_SECRET); - } - return null; - }); - } + authenticateUsingBasic(BASIC_REALM, user -> { + final String configUser = config.getString(Key.API_USERNAME, ""); + if (!configUser.isEmpty() && configUser.equals(user)) { + return config.getString(Key.API_SECRET); + } + return null; + }); } - abstract boolean requiresAuth(); - } diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/api/ConfigurationResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/api/ConfigurationResource.java index b1f223c88..bed8838fe 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/api/ConfigurationResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/api/ConfigurationResource.java @@ -77,9 +77,4 @@ public void doPUT() throws IOException { getResponse().setStatus(Status.NO_CONTENT.getCode()); } - @Override - boolean requiresAuth() { - return true; - } - } diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/api/StatusResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/api/StatusResource.java index 6feeb8d0f..1ed1bf0ff 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/api/StatusResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/api/StatusResource.java @@ -43,9 +43,4 @@ public void doGET() throws IOException { .write(getResponse().getOutputStream(), features); } - @Override - boolean requiresAuth() { - return true; - } - } diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/api/TaskResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/api/TaskResource.java index dc5ab6af9..84deec047 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/api/TaskResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/api/TaskResource.java @@ -52,9 +52,4 @@ public void doGET() throws Exception { } } - @Override - boolean requiresAuth() { - return true; - } - } diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/api/TasksResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/api/TasksResource.java index 4e0009df5..aae5ffe80 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/api/TasksResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/api/TasksResource.java @@ -74,9 +74,4 @@ public void doPOST() throws Exception { } } - @Override - boolean requiresAuth() { - return true; - } - } From 30c8c8dcf5b547aeac9b0aa62bc663e757101a6c Mon Sep 17 00:00:00 2001 From: Dave C Date: Fri, 2 Jul 2021 16:00:22 -0400 Subject: [PATCH 010/106] Update jetty version. resolves #411 (#516) --- pom.xml | 2 +- .../ApplicationContextListener.java | 4 +-- .../library/cantaloupe/ApplicationServer.java | 7 +++-- .../library/cantaloupe/http/Server.java | 5 +++- .../codec/IIOProviderContextListener.java | 4 +-- .../cantaloupe/resource/AbstractResource.java | 4 +-- .../cantaloupe/resource/FileServlet.java | 6 ++--- .../cantaloupe/resource/HandlerServlet.java | 6 ++--- .../library/cantaloupe/resource/Request.java | 2 +- .../library/cantaloupe/http/ServerTest.java | 4 +-- .../ByteArrayServletOutputStream.java | 4 +-- .../resource/MockHttpServletRequest.java | 26 +++++++++---------- .../resource/MockHttpServletResponse.java | 4 +-- .../source/HTTPStreamFactoryTest.java | 5 ++-- .../cantaloupe/source/HttpSourceTest.java | 4 +-- .../OkHttpHTTPImageInputStreamClientTest.java | 4 +-- 16 files changed, 48 insertions(+), 43 deletions(-) diff --git a/pom.xml b/pom.xml index b4b575321..70ced69d2 100644 --- a/pom.xml +++ b/pom.xml @@ -17,7 +17,7 @@ UTF-8 2.15.28 2.11.0 - 9.4.34.v20201102 + 11.0.5 9.2.17.0 3.0.0-M3 diff --git a/src/main/java/edu/illinois/library/cantaloupe/ApplicationContextListener.java b/src/main/java/edu/illinois/library/cantaloupe/ApplicationContextListener.java index 9db7b83e9..0071991a2 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/ApplicationContextListener.java +++ b/src/main/java/edu/illinois/library/cantaloupe/ApplicationContextListener.java @@ -13,9 +13,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import jakarta.servlet.ServletContextEvent; +import jakarta.servlet.ServletContextListener; import javax.script.ScriptEngineManager; -import javax.servlet.ServletContextEvent; -import javax.servlet.ServletContextListener; import java.util.stream.Collectors; /** diff --git a/src/main/java/edu/illinois/library/cantaloupe/ApplicationServer.java b/src/main/java/edu/illinois/library/cantaloupe/ApplicationServer.java index a048842a0..a35c07c7f 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/ApplicationServer.java +++ b/src/main/java/edu/illinois/library/cantaloupe/ApplicationServer.java @@ -6,6 +6,7 @@ import edu.illinois.library.cantaloupe.resource.FileServlet; import edu.illinois.library.cantaloupe.resource.HandlerServlet; import org.eclipse.jetty.alpn.server.ALPNServerConnectionFactory; +import org.eclipse.jetty.http.UriCompliance; import org.eclipse.jetty.http2.HTTP2Cipher; import org.eclipse.jetty.http2.server.HTTP2CServerConnectionFactory; import org.eclipse.jetty.http2.server.HTTP2ServerConnectionFactory; @@ -265,7 +266,8 @@ public void start() throws Exception { // HTTP/2. if (isHTTPEnabled()) { HttpConfiguration config = new HttpConfiguration(); - HttpConnectionFactory http1 = new HttpConnectionFactory(); + config.setUriCompliance(UriCompliance.LEGACY); + HttpConnectionFactory http1 = new HttpConnectionFactory(config); HTTP2CServerConnectionFactory http2 = new HTTP2CServerConnectionFactory(config); @@ -280,11 +282,12 @@ public void start() throws Exception { // Initialize the HTTPS server. if (isHTTPSEnabled()) { HttpConfiguration config = new HttpConfiguration(); + config.setUriCompliance(UriCompliance.LEGACY); config.setSecureScheme("https"); config.setSecurePort(getHTTPSPort()); config.addCustomizer(new SecureRequestCustomizer()); - final SslContextFactory contextFactory = new SslContextFactory.Server(); + final SslContextFactory.Server contextFactory = new SslContextFactory.Server(); contextFactory.setKeyStorePath(getHTTPSKeyStorePath()); if (getHTTPSKeyStorePassword() != null) { contextFactory.setKeyStorePassword(getHTTPSKeyStorePassword()); diff --git a/src/main/java/edu/illinois/library/cantaloupe/http/Server.java b/src/main/java/edu/illinois/library/cantaloupe/http/Server.java index 21c9b37e3..ba049467e 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/http/Server.java +++ b/src/main/java/edu/illinois/library/cantaloupe/http/Server.java @@ -2,6 +2,7 @@ import edu.illinois.library.cantaloupe.util.SocketUtils; import org.eclipse.jetty.alpn.server.ALPNServerConnectionFactory; +import org.eclipse.jetty.http.UriCompliance; import org.eclipse.jetty.http2.HTTP2Cipher; import org.eclipse.jetty.http2.server.HTTP2CServerConnectionFactory; import org.eclipse.jetty.http2.server.HTTP2ServerConnectionFactory; @@ -73,6 +74,7 @@ private void initializeServer() { ServerConnector connector; HttpConfiguration config = new HttpConfiguration(); + config.setUriCompliance(UriCompliance.LEGACY); HttpConnectionFactory http1 = new HttpConnectionFactory(config); HTTP2CServerConnectionFactory http2c = @@ -95,10 +97,11 @@ private void initializeServer() { // Initialize HTTPS. if (isHTTPS1Enabled || isHTTPS2Enabled) { config = new HttpConfiguration(); + config.setUriCompliance(UriCompliance.LEGACY); config.setSecureScheme("https"); config.addCustomizer(new SecureRequestCustomizer()); - final SslContextFactory contextFactory = + final SslContextFactory.Server contextFactory = new SslContextFactory.Server(); contextFactory.setKeyStorePath(keyStorePath.toString()); contextFactory.setKeyStorePassword(keyStorePassword); diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/IIOProviderContextListener.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/IIOProviderContextListener.java index f575915e6..209ed6028 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/IIOProviderContextListener.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/IIOProviderContextListener.java @@ -4,11 +4,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import jakarta.servlet.ServletContextEvent; +import jakarta.servlet.ServletContextListener; import javax.imageio.ImageIO; import javax.imageio.spi.IIORegistry; import javax.imageio.spi.ServiceRegistry; -import javax.servlet.ServletContextEvent; -import javax.servlet.ServletContextListener; import java.util.ArrayList; import java.util.Iterator; import java.util.List; diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/AbstractResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/AbstractResource.java index 20181b298..7b2c04e8d 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/AbstractResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/AbstractResource.java @@ -21,7 +21,7 @@ import edu.illinois.library.cantaloupe.util.StringUtils; import org.slf4j.Logger; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; @@ -41,7 +41,7 @@ * more of the HTTP-method-specific methods {@link #doGET()} etc., and may * optionally use {@link #doInit()} and {@link #destroy()}.

      * - *

      Unlike {@link javax.servlet.http.HttpServlet}s, instances are only used + *

      Unlike {@link jakarta.servlet.http.HttpServlet}s, instances are only used * once and not shared across threads.

      */ public abstract class AbstractResource { diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/FileServlet.java b/src/main/java/edu/illinois/library/cantaloupe/resource/FileServlet.java index cbad65f02..94551c415 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/FileServlet.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/FileServlet.java @@ -1,8 +1,8 @@ package edu.illinois.library.cantaloupe.resource; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/HandlerServlet.java b/src/main/java/edu/illinois/library/cantaloupe/resource/HandlerServlet.java index 2223870ec..b84025a3d 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/HandlerServlet.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/HandlerServlet.java @@ -8,9 +8,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.PrintWriter; import java.util.List; diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/Request.java b/src/main/java/edu/illinois/library/cantaloupe/resource/Request.java index 55588eab4..9e36c9a38 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/Request.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/Request.java @@ -6,7 +6,7 @@ import edu.illinois.library.cantaloupe.http.Query; import edu.illinois.library.cantaloupe.http.Reference; -import javax.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletRequest; import java.io.IOException; import java.io.InputStream; import java.util.Enumeration; diff --git a/src/test/java/edu/illinois/library/cantaloupe/http/ServerTest.java b/src/test/java/edu/illinois/library/cantaloupe/http/ServerTest.java index 84c79a215..1e6002d58 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/http/ServerTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/http/ServerTest.java @@ -9,8 +9,8 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import static org.junit.jupiter.api.Assertions.*; diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/ByteArrayServletOutputStream.java b/src/test/java/edu/illinois/library/cantaloupe/resource/ByteArrayServletOutputStream.java index 99f8486ec..123bc2120 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/ByteArrayServletOutputStream.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/ByteArrayServletOutputStream.java @@ -1,7 +1,7 @@ package edu.illinois.library.cantaloupe.resource; -import javax.servlet.ServletOutputStream; -import javax.servlet.WriteListener; +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.WriteListener; import java.io.ByteArrayOutputStream; import java.io.IOException; diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletRequest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletRequest.java index ef6d5926a..1bdb796ab 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletRequest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletRequest.java @@ -1,18 +1,18 @@ package edu.illinois.library.cantaloupe.resource; -import javax.servlet.AsyncContext; -import javax.servlet.DispatcherType; -import javax.servlet.RequestDispatcher; -import javax.servlet.ServletContext; -import javax.servlet.ServletInputStream; -import javax.servlet.ServletRequest; -import javax.servlet.ServletResponse; -import javax.servlet.http.Cookie; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.servlet.http.HttpSession; -import javax.servlet.http.HttpUpgradeHandler; -import javax.servlet.http.Part; +import jakarta.servlet.AsyncContext; +import jakarta.servlet.DispatcherType; +import jakarta.servlet.RequestDispatcher; +import jakarta.servlet.ServletContext; +import jakarta.servlet.ServletInputStream; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import jakarta.servlet.http.Cookie; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpSession; +import jakarta.servlet.http.HttpUpgradeHandler; +import jakarta.servlet.http.Part; import java.io.BufferedReader; import java.security.Principal; import java.util.Collection; diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletResponse.java b/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletResponse.java index 749e314f3..638cd4599 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletResponse.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletResponse.java @@ -3,8 +3,8 @@ import edu.illinois.library.cantaloupe.http.Header; import edu.illinois.library.cantaloupe.http.Headers; -import javax.servlet.http.Cookie; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.Cookie; +import jakarta.servlet.http.HttpServletResponse; import java.io.PrintWriter; import java.util.Collection; import java.util.Locale; diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/HTTPStreamFactoryTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/HTTPStreamFactoryTest.java index 9dc1ec791..191ac9c62 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/HTTPStreamFactoryTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/HTTPStreamFactoryTest.java @@ -13,11 +13,10 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import javax.imageio.stream.ImageInputStream; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.io.InputStream; -import java.util.HashMap; import java.util.Map; import static org.junit.jupiter.api.Assertions.*; diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java index 065455c4d..cd1ad104d 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java @@ -15,8 +15,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/OkHttpHTTPImageInputStreamClientTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/OkHttpHTTPImageInputStreamClientTest.java index 40617ea81..02a362a08 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/OkHttpHTTPImageInputStreamClientTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/OkHttpHTTPImageInputStreamClientTest.java @@ -10,8 +10,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import static org.junit.jupiter.api.Assertions.*; From 0e186d74120606ba5b3678e91035f09a609984b0 Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Fri, 2 Jul 2021 15:08:36 -0500 Subject: [PATCH 011/106] Rename CompletableOutputStream.isCompletelyWritten() to isComplete() and setCompletelyWritten() to complete() --- .../cantaloupe/cache/AzureStorageCache.java | 4 +-- .../cache/CompletableOutputStream.java | 8 +++--- .../cantaloupe/cache/DerivativeCache.java | 2 +- .../cantaloupe/cache/FilesystemCache.java | 4 +-- .../library/cantaloupe/cache/HeapCache.java | 2 +- .../library/cantaloupe/cache/JdbcCache.java | 4 +-- .../library/cantaloupe/cache/RedisCache.java | 2 +- .../library/cantaloupe/cache/S3Cache.java | 2 +- .../resource/ImageRepresentation.java | 2 +- .../cantaloupe/cache/AbstractCacheTest.java | 26 +++++++++---------- .../cantaloupe/cache/CacheFacadeTest.java | 14 +++++----- .../cantaloupe/cache/HeapCacheTest.java | 10 +++---- .../cantaloupe/cache/JdbcCacheTest.java | 6 ++--- .../library/cantaloupe/cache/S3CacheTest.java | 2 +- .../resource/ImageRequestHandlerTest.java | 4 +-- 15 files changed, 46 insertions(+), 46 deletions(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java index b6cf5082f..1ba50b610 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java @@ -67,7 +67,7 @@ private static class CustomBlobOutputStream /** * Constructor for an instance that writes into the given temporary - * blob. Upon closure, if the stream is {@link #isCompletelyWritten() + * blob. Upon closure, if the stream is {@link #isComplete() * completely written}, the temporary blob is copied into place and * deleted. Otherwise, the temporary blob is deleted. * @@ -96,7 +96,7 @@ public void close() throws IOException { blobOutputStream.flush(); blobOutputStream.close(); if (container != null) { - if (isCompletelyWritten()) { + if (isComplete()) { // Copy the temporary blob into place. CloudBlockBlob destBlob = container.getBlockBlobReference(blobKey); diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/CompletableOutputStream.java b/src/main/java/edu/illinois/library/cantaloupe/cache/CompletableOutputStream.java index a0d9b3f98..4134d4b9c 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/CompletableOutputStream.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/CompletableOutputStream.java @@ -12,12 +12,12 @@ public abstract class CompletableOutputStream extends OutputStream { private boolean isCompletelyWritten; - public boolean isCompletelyWritten() { - return isCompletelyWritten; + public void complete() { + this.isCompletelyWritten = true; } - public void setCompletelyWritten(boolean isCompletelyWritten) { - this.isCompletelyWritten = isCompletelyWritten; + public boolean isComplete() { + return isCompletelyWritten; } } diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java index 8ce1b7f6a..cd03641f8 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java @@ -57,7 +57,7 @@ InputStream newDerivativeImageInputStream(OperationList opList) * *

      The {@link CompletableOutputStream#close()} method of the returned * instance must check the return value of {@link - * CompletableOutputStream#isCompletelyWritten()} before committing data + * CompletableOutputStream#isComplete()} before committing data * to the cache. If it returns {@code false}, any written data should be * discarded.

      * diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java index 6fd9110e7..2656465dc 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java @@ -181,7 +181,7 @@ public void close() { // If the written file is complete, move it into place. // Otherwise, delete it. - if (isCompletelyWritten()) { + if (isComplete()) { CFOS_LOGGER.debug("close(): moving {} to {}", tempFile, destinationFile); Files.move(tempFile, destinationFile, @@ -634,7 +634,7 @@ identifier, sourceImageTempFile(identifier), // work with newDerivativeImageOutputStream(). But this method does not // need that extra functionality, so setting it as completely written // here makes it behave like an ordinary OutputStream. - os.setCompletelyWritten(true); + os.complete(); return os; } diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/HeapCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/HeapCache.java index acb869843..f870565e0 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/HeapCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/HeapCache.java @@ -184,7 +184,7 @@ private class HeapCacheOutputStream extends CompletableOutputStream { @Override public void close() throws IOException { LOGGER.debug("Closing stream for {}", opList); - if (isCompletelyWritten()) { + if (isComplete()) { Key key = itemKey(opList); Item item = new Item(wrappedStream.toByteArray()); cache.put(key, item); diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java index 01d6fe637..f147f0700 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java @@ -48,7 +48,7 @@ class JdbcCache implements DerivativeCache { /** * Wraps a {@link Blob} OutputStream, for writing an image to a BLOB. * The constructor creates a transaction that is committed on close if the - * stream is {@link CompletableOutputStream#isCompletelyWritten() + * stream is {@link CompletableOutputStream#isComplete() * completely written}. */ private class ImageBlobOutputStream extends CompletableOutputStream { @@ -92,7 +92,7 @@ private class ImageBlobOutputStream extends CompletableOutputStream { public void close() throws IOException { LOGGER.debug("Closing stream for {}", ops); try { - if (isCompletelyWritten()) { + if (isComplete()) { statement.executeUpdate(); connection.commit(); } else { diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/RedisCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/RedisCache.java index bb0c2053e..6996fde1b 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/RedisCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/RedisCache.java @@ -156,7 +156,7 @@ private static class RedisOutputStream extends CompletableOutputStream { @Override public void close() throws IOException { try { - if (isCompletelyWritten()) { + if (isComplete()) { connection.async().hset(hashKey, valueKey, bufferStream.toByteArray()); } diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java index 258d53f2c..c69b60c57 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java @@ -116,7 +116,7 @@ public void close() throws IOException { try { bufferStream.close(); byte[] data = bufferStream.toByteArray(); - if (isCompletelyWritten()) { + if (isComplete()) { // At this point, the client has received all image data, // but it is still waiting for the connection to close. // Uploading in a separate thread will allow this to happen diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRepresentation.java b/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRepresentation.java index d05cd3fa8..c6fc18023 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRepresentation.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRepresentation.java @@ -129,7 +129,7 @@ public void write(OutputStream responseOS) throws IOException { "cache simultaneously"); copyOrProcess(teeOS); cacheOS.flush(); - cacheOS.setCompletelyWritten(true); + cacheOS.complete(); } else { copyOrProcess(responseOS); } diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java index f1c35b330..be35eed59 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java @@ -110,7 +110,7 @@ void testNewDerivativeImageInputStreamWithZeroTTL() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList)) { Files.copy(imageFile, os); - os.setCompletelyWritten(true); + os.complete(); } // Wait for it to upload @@ -144,7 +144,7 @@ void testNewDerivativeImageInputStreamWithNonzeroTTL() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(fixture, os); - os.setCompletelyWritten(true); + os.complete(); } // Wait for it to finish, hopefully. @@ -182,7 +182,7 @@ void testNewDerivativeImageInputStreamConcurrently() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.complete(); } return null; }, () -> { @@ -216,7 +216,7 @@ void testNewDerivativeImageOutputStream() throws Exception { try (CompletableOutputStream outputStream = instance.newDerivativeImageOutputStream(ops)) { Files.copy(fixture, outputStream); - outputStream.setCompletelyWritten(true); + outputStream.complete(); } // Wait for it to upload @@ -245,7 +245,7 @@ void testNewDerivativeImageOutputStreamDoesNotLeaveDetritusWhenStreamIsIncomplet try (CompletableOutputStream outputStream = instance.newDerivativeImageOutputStream(ops)) { Files.copy(fixture, outputStream); - outputStream.setCompletelyWritten(false); // the whole point of the test + outputStream.complete(); // the whole point of the test } // Wait for it to upload @@ -292,7 +292,7 @@ void testPurge() throws Exception { instance.newDerivativeImageOutputStream(opList)) { Path fixture = TestUtil.getImage(IMAGE); Files.copy(fixture, outputStream); - outputStream.setCompletelyWritten(true); + outputStream.complete(); } // add the info @@ -332,7 +332,7 @@ void testPurgeWithIdentifier() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList1)) { Files.copy(fixture, os); - os.setCompletelyWritten(true); + os.complete(); } instance.put(id1, new Info()); @@ -345,7 +345,7 @@ void testPurgeWithIdentifier() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList2)) { Files.copy(fixture, os); - os.setCompletelyWritten(true); + os.complete(); } instance.put(id2, new Info()); @@ -385,7 +385,7 @@ void testPurgeWithOperationList() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops1)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.complete(); } // Seed another derivative image @@ -396,7 +396,7 @@ void testPurgeWithOperationList() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops2)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.complete(); } Thread.sleep(ASYNC_WAIT); @@ -436,7 +436,7 @@ void testPurgeInfos() throws Exception { instance.newDerivativeImageOutputStream(opList)) { Path fixture = TestUtil.getImage(IMAGE); Files.copy(fixture, outputStream); - outputStream.setCompletelyWritten(true); + outputStream.complete(); } // add the info @@ -476,7 +476,7 @@ void testPurgeInvalid() throws Exception { try (CompletableOutputStream outputStream = instance.newDerivativeImageOutputStream(ops1)) { Files.copy(fixture, outputStream); - outputStream.setCompletelyWritten(true); + outputStream.complete(); } // add an Info @@ -499,7 +499,7 @@ void testPurgeInvalid() throws Exception { try (CompletableOutputStream outputStream = instance.newDerivativeImageOutputStream(ops2)) { Files.copy(fixture2, outputStream); - outputStream.setCompletelyWritten(true); + outputStream.complete(); } // add another info diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/CacheFacadeTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/CacheFacadeTest.java index d1350362e..42e1160fe 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/CacheFacadeTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/CacheFacadeTest.java @@ -189,7 +189,7 @@ void testNewDerivativeImageInputStreamWhenDerivativeCacheIsEnabled() try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList)) { Files.copy(TestUtil.getImage("jpg"), os); - os.setCompletelyWritten(true); + os.complete(); } try (InputStream is = instance.newDerivativeImageInputStream(opList)) { @@ -217,7 +217,7 @@ void testNewDerivativeImageOutputStreamWhenDerivativeCacheIsEnabled() try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList)) { assertNotNull(os); - os.setCompletelyWritten(true); + os.complete(); } } @@ -253,7 +253,7 @@ void testPurge() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage("jpg"), os); - os.setCompletelyWritten(true); + os.complete(); } // Add info to the derivative cache. @@ -299,7 +299,7 @@ void testPurgeWithIdentifier() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage("jpg"), os); - os.setCompletelyWritten(true); + os.complete(); } // Add info to the derivative cache. @@ -343,7 +343,7 @@ void testPurgeAsyncWithIdentifier() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage("jpg"), os); - os.setCompletelyWritten(true); + os.complete(); } // Add info to the derivative cache. @@ -378,7 +378,7 @@ void testPurgeWithOperationList() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList)) { Files.copy(TestUtil.getImage("jpg"), os); - os.setCompletelyWritten(true); + os.complete(); } try (InputStream is = instance.newDerivativeImageInputStream(opList)) { @@ -447,7 +447,7 @@ void testPurgeInvalid() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage("jpg"), os); - os.setCompletelyWritten(true); + os.complete(); } // Add info to the derivative cache. diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/HeapCacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/HeapCacheTest.java index 42a73409d..a052670ef 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/HeapCacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/HeapCacheTest.java @@ -98,7 +98,7 @@ void testDumpToPersistentStore() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.complete(); } instance.dumpToPersistentStore(); @@ -127,7 +127,7 @@ void testGetByteSize() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.complete(); } assertEquals(5439, instance.getByteSize()); @@ -231,7 +231,7 @@ void testLoadFromPersistentStore() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.complete(); } instance.dumpToPersistentStore(); @@ -296,7 +296,7 @@ void testPurgeExcessWithExcess() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.complete(); } assertEquals(5439, instance.getByteSize()); @@ -317,7 +317,7 @@ void testPurgeExcessWithNoExcess() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.complete(); } long size = instance.getByteSize(); diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/JdbcCacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/JdbcCacheTest.java index edbfdb998..7d01bd8b0 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/JdbcCacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/JdbcCacheTest.java @@ -106,7 +106,7 @@ private void seed(Connection connection) throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.complete(); } Crop crop = new CropByPixels(50, 50, 50, 50); @@ -119,7 +119,7 @@ private void seed(Connection connection) throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.complete(); } crop = new CropByPixels(10, 20, 50, 90); @@ -132,7 +132,7 @@ private void seed(Connection connection) throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.complete(); } // persist some infos corresponding to the above images diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java index 80920af13..445aa89cf 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java @@ -242,7 +242,7 @@ void testNewDerivativeImageInputStreamUpdatesLastModifiedTime() try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(fixture, os); - os.setCompletelyWritten(true); + os.complete(); } // Wait for it to finish, hopefully. diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandlerTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandlerTest.java index 641f9e518..17d826718 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandlerTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandlerTest.java @@ -218,7 +218,7 @@ void testHandleCallsCacheStreamingCallback() throws Exception { try (CompletableOutputStream os = cache.newDerivativeImageOutputStream(opList)) { os.write(new byte[] { 0x35, 0x35, 0x35 }); - os.setCompletelyWritten(true); + os.complete(); } final IntrospectiveCallback callback = new IntrospectiveCallback(); @@ -348,7 +348,7 @@ void testHandleStreamsFromDerivativeCache() throws Exception { try (CompletableOutputStream os = cache.newDerivativeImageOutputStream(opList)) { os.write(expected); - os.setCompletelyWritten(true); + os.complete(); } final IntrospectiveCallback callback = new IntrospectiveCallback(); From ddbfd5e27b7d3489eaedec725cea61e4a4b2a15d Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Fri, 2 Jul 2021 15:43:44 -0500 Subject: [PATCH 012/106] Revert "Rename CompletableOutputStream.isCompletelyWritten() to isComplete() and setCompletelyWritten() to complete()" This reverts commit 0e186d74120606ba5b3678e91035f09a609984b0. --- .../cantaloupe/cache/AzureStorageCache.java | 4 +-- .../cache/CompletableOutputStream.java | 8 +++--- .../cantaloupe/cache/DerivativeCache.java | 2 +- .../cantaloupe/cache/FilesystemCache.java | 4 +-- .../library/cantaloupe/cache/HeapCache.java | 2 +- .../library/cantaloupe/cache/JdbcCache.java | 4 +-- .../library/cantaloupe/cache/RedisCache.java | 2 +- .../library/cantaloupe/cache/S3Cache.java | 2 +- .../resource/ImageRepresentation.java | 2 +- .../cantaloupe/cache/AbstractCacheTest.java | 26 +++++++++---------- .../cantaloupe/cache/CacheFacadeTest.java | 14 +++++----- .../cantaloupe/cache/HeapCacheTest.java | 10 +++---- .../cantaloupe/cache/JdbcCacheTest.java | 6 ++--- .../library/cantaloupe/cache/S3CacheTest.java | 2 +- .../resource/ImageRequestHandlerTest.java | 4 +-- 15 files changed, 46 insertions(+), 46 deletions(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java index 1ba50b610..b6cf5082f 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java @@ -67,7 +67,7 @@ private static class CustomBlobOutputStream /** * Constructor for an instance that writes into the given temporary - * blob. Upon closure, if the stream is {@link #isComplete() + * blob. Upon closure, if the stream is {@link #isCompletelyWritten() * completely written}, the temporary blob is copied into place and * deleted. Otherwise, the temporary blob is deleted. * @@ -96,7 +96,7 @@ public void close() throws IOException { blobOutputStream.flush(); blobOutputStream.close(); if (container != null) { - if (isComplete()) { + if (isCompletelyWritten()) { // Copy the temporary blob into place. CloudBlockBlob destBlob = container.getBlockBlobReference(blobKey); diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/CompletableOutputStream.java b/src/main/java/edu/illinois/library/cantaloupe/cache/CompletableOutputStream.java index 4134d4b9c..a0d9b3f98 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/CompletableOutputStream.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/CompletableOutputStream.java @@ -12,12 +12,12 @@ public abstract class CompletableOutputStream extends OutputStream { private boolean isCompletelyWritten; - public void complete() { - this.isCompletelyWritten = true; + public boolean isCompletelyWritten() { + return isCompletelyWritten; } - public boolean isComplete() { - return isCompletelyWritten; + public void setCompletelyWritten(boolean isCompletelyWritten) { + this.isCompletelyWritten = isCompletelyWritten; } } diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java index cd03641f8..8ce1b7f6a 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java @@ -57,7 +57,7 @@ InputStream newDerivativeImageInputStream(OperationList opList) * *

      The {@link CompletableOutputStream#close()} method of the returned * instance must check the return value of {@link - * CompletableOutputStream#isComplete()} before committing data + * CompletableOutputStream#isCompletelyWritten()} before committing data * to the cache. If it returns {@code false}, any written data should be * discarded.

      * diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java index 2656465dc..6fd9110e7 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java @@ -181,7 +181,7 @@ public void close() { // If the written file is complete, move it into place. // Otherwise, delete it. - if (isComplete()) { + if (isCompletelyWritten()) { CFOS_LOGGER.debug("close(): moving {} to {}", tempFile, destinationFile); Files.move(tempFile, destinationFile, @@ -634,7 +634,7 @@ identifier, sourceImageTempFile(identifier), // work with newDerivativeImageOutputStream(). But this method does not // need that extra functionality, so setting it as completely written // here makes it behave like an ordinary OutputStream. - os.complete(); + os.setCompletelyWritten(true); return os; } diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/HeapCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/HeapCache.java index f870565e0..acb869843 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/HeapCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/HeapCache.java @@ -184,7 +184,7 @@ private class HeapCacheOutputStream extends CompletableOutputStream { @Override public void close() throws IOException { LOGGER.debug("Closing stream for {}", opList); - if (isComplete()) { + if (isCompletelyWritten()) { Key key = itemKey(opList); Item item = new Item(wrappedStream.toByteArray()); cache.put(key, item); diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java index f147f0700..01d6fe637 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java @@ -48,7 +48,7 @@ class JdbcCache implements DerivativeCache { /** * Wraps a {@link Blob} OutputStream, for writing an image to a BLOB. * The constructor creates a transaction that is committed on close if the - * stream is {@link CompletableOutputStream#isComplete() + * stream is {@link CompletableOutputStream#isCompletelyWritten() * completely written}. */ private class ImageBlobOutputStream extends CompletableOutputStream { @@ -92,7 +92,7 @@ private class ImageBlobOutputStream extends CompletableOutputStream { public void close() throws IOException { LOGGER.debug("Closing stream for {}", ops); try { - if (isComplete()) { + if (isCompletelyWritten()) { statement.executeUpdate(); connection.commit(); } else { diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/RedisCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/RedisCache.java index 6996fde1b..bb0c2053e 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/RedisCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/RedisCache.java @@ -156,7 +156,7 @@ private static class RedisOutputStream extends CompletableOutputStream { @Override public void close() throws IOException { try { - if (isComplete()) { + if (isCompletelyWritten()) { connection.async().hset(hashKey, valueKey, bufferStream.toByteArray()); } diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java index c69b60c57..258d53f2c 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java @@ -116,7 +116,7 @@ public void close() throws IOException { try { bufferStream.close(); byte[] data = bufferStream.toByteArray(); - if (isComplete()) { + if (isCompletelyWritten()) { // At this point, the client has received all image data, // but it is still waiting for the connection to close. // Uploading in a separate thread will allow this to happen diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRepresentation.java b/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRepresentation.java index c6fc18023..d05cd3fa8 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRepresentation.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRepresentation.java @@ -129,7 +129,7 @@ public void write(OutputStream responseOS) throws IOException { "cache simultaneously"); copyOrProcess(teeOS); cacheOS.flush(); - cacheOS.complete(); + cacheOS.setCompletelyWritten(true); } else { copyOrProcess(responseOS); } diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java index be35eed59..f1c35b330 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java @@ -110,7 +110,7 @@ void testNewDerivativeImageInputStreamWithZeroTTL() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList)) { Files.copy(imageFile, os); - os.complete(); + os.setCompletelyWritten(true); } // Wait for it to upload @@ -144,7 +144,7 @@ void testNewDerivativeImageInputStreamWithNonzeroTTL() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(fixture, os); - os.complete(); + os.setCompletelyWritten(true); } // Wait for it to finish, hopefully. @@ -182,7 +182,7 @@ void testNewDerivativeImageInputStreamConcurrently() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.complete(); + os.setCompletelyWritten(true); } return null; }, () -> { @@ -216,7 +216,7 @@ void testNewDerivativeImageOutputStream() throws Exception { try (CompletableOutputStream outputStream = instance.newDerivativeImageOutputStream(ops)) { Files.copy(fixture, outputStream); - outputStream.complete(); + outputStream.setCompletelyWritten(true); } // Wait for it to upload @@ -245,7 +245,7 @@ void testNewDerivativeImageOutputStreamDoesNotLeaveDetritusWhenStreamIsIncomplet try (CompletableOutputStream outputStream = instance.newDerivativeImageOutputStream(ops)) { Files.copy(fixture, outputStream); - outputStream.complete(); // the whole point of the test + outputStream.setCompletelyWritten(false); // the whole point of the test } // Wait for it to upload @@ -292,7 +292,7 @@ void testPurge() throws Exception { instance.newDerivativeImageOutputStream(opList)) { Path fixture = TestUtil.getImage(IMAGE); Files.copy(fixture, outputStream); - outputStream.complete(); + outputStream.setCompletelyWritten(true); } // add the info @@ -332,7 +332,7 @@ void testPurgeWithIdentifier() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList1)) { Files.copy(fixture, os); - os.complete(); + os.setCompletelyWritten(true); } instance.put(id1, new Info()); @@ -345,7 +345,7 @@ void testPurgeWithIdentifier() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList2)) { Files.copy(fixture, os); - os.complete(); + os.setCompletelyWritten(true); } instance.put(id2, new Info()); @@ -385,7 +385,7 @@ void testPurgeWithOperationList() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops1)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.complete(); + os.setCompletelyWritten(true); } // Seed another derivative image @@ -396,7 +396,7 @@ void testPurgeWithOperationList() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops2)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.complete(); + os.setCompletelyWritten(true); } Thread.sleep(ASYNC_WAIT); @@ -436,7 +436,7 @@ void testPurgeInfos() throws Exception { instance.newDerivativeImageOutputStream(opList)) { Path fixture = TestUtil.getImage(IMAGE); Files.copy(fixture, outputStream); - outputStream.complete(); + outputStream.setCompletelyWritten(true); } // add the info @@ -476,7 +476,7 @@ void testPurgeInvalid() throws Exception { try (CompletableOutputStream outputStream = instance.newDerivativeImageOutputStream(ops1)) { Files.copy(fixture, outputStream); - outputStream.complete(); + outputStream.setCompletelyWritten(true); } // add an Info @@ -499,7 +499,7 @@ void testPurgeInvalid() throws Exception { try (CompletableOutputStream outputStream = instance.newDerivativeImageOutputStream(ops2)) { Files.copy(fixture2, outputStream); - outputStream.complete(); + outputStream.setCompletelyWritten(true); } // add another info diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/CacheFacadeTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/CacheFacadeTest.java index 42e1160fe..d1350362e 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/CacheFacadeTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/CacheFacadeTest.java @@ -189,7 +189,7 @@ void testNewDerivativeImageInputStreamWhenDerivativeCacheIsEnabled() try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList)) { Files.copy(TestUtil.getImage("jpg"), os); - os.complete(); + os.setCompletelyWritten(true); } try (InputStream is = instance.newDerivativeImageInputStream(opList)) { @@ -217,7 +217,7 @@ void testNewDerivativeImageOutputStreamWhenDerivativeCacheIsEnabled() try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList)) { assertNotNull(os); - os.complete(); + os.setCompletelyWritten(true); } } @@ -253,7 +253,7 @@ void testPurge() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage("jpg"), os); - os.complete(); + os.setCompletelyWritten(true); } // Add info to the derivative cache. @@ -299,7 +299,7 @@ void testPurgeWithIdentifier() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage("jpg"), os); - os.complete(); + os.setCompletelyWritten(true); } // Add info to the derivative cache. @@ -343,7 +343,7 @@ void testPurgeAsyncWithIdentifier() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage("jpg"), os); - os.complete(); + os.setCompletelyWritten(true); } // Add info to the derivative cache. @@ -378,7 +378,7 @@ void testPurgeWithOperationList() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList)) { Files.copy(TestUtil.getImage("jpg"), os); - os.complete(); + os.setCompletelyWritten(true); } try (InputStream is = instance.newDerivativeImageInputStream(opList)) { @@ -447,7 +447,7 @@ void testPurgeInvalid() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage("jpg"), os); - os.complete(); + os.setCompletelyWritten(true); } // Add info to the derivative cache. diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/HeapCacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/HeapCacheTest.java index a052670ef..42a73409d 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/HeapCacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/HeapCacheTest.java @@ -98,7 +98,7 @@ void testDumpToPersistentStore() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.complete(); + os.setCompletelyWritten(true); } instance.dumpToPersistentStore(); @@ -127,7 +127,7 @@ void testGetByteSize() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.complete(); + os.setCompletelyWritten(true); } assertEquals(5439, instance.getByteSize()); @@ -231,7 +231,7 @@ void testLoadFromPersistentStore() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.complete(); + os.setCompletelyWritten(true); } instance.dumpToPersistentStore(); @@ -296,7 +296,7 @@ void testPurgeExcessWithExcess() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.complete(); + os.setCompletelyWritten(true); } assertEquals(5439, instance.getByteSize()); @@ -317,7 +317,7 @@ void testPurgeExcessWithNoExcess() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.complete(); + os.setCompletelyWritten(true); } long size = instance.getByteSize(); diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/JdbcCacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/JdbcCacheTest.java index 7d01bd8b0..edbfdb998 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/JdbcCacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/JdbcCacheTest.java @@ -106,7 +106,7 @@ private void seed(Connection connection) throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.complete(); + os.setCompletelyWritten(true); } Crop crop = new CropByPixels(50, 50, 50, 50); @@ -119,7 +119,7 @@ private void seed(Connection connection) throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.complete(); + os.setCompletelyWritten(true); } crop = new CropByPixels(10, 20, 50, 90); @@ -132,7 +132,7 @@ private void seed(Connection connection) throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.complete(); + os.setCompletelyWritten(true); } // persist some infos corresponding to the above images diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java index 445aa89cf..80920af13 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java @@ -242,7 +242,7 @@ void testNewDerivativeImageInputStreamUpdatesLastModifiedTime() try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(fixture, os); - os.complete(); + os.setCompletelyWritten(true); } // Wait for it to finish, hopefully. diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandlerTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandlerTest.java index 17d826718..641f9e518 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandlerTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandlerTest.java @@ -218,7 +218,7 @@ void testHandleCallsCacheStreamingCallback() throws Exception { try (CompletableOutputStream os = cache.newDerivativeImageOutputStream(opList)) { os.write(new byte[] { 0x35, 0x35, 0x35 }); - os.complete(); + os.setCompletelyWritten(true); } final IntrospectiveCallback callback = new IntrospectiveCallback(); @@ -348,7 +348,7 @@ void testHandleStreamsFromDerivativeCache() throws Exception { try (CompletableOutputStream os = cache.newDerivativeImageOutputStream(opList)) { os.write(expected); - os.complete(); + os.setCompletelyWritten(true); } final IntrospectiveCallback callback = new IntrospectiveCallback(); From 89be4b3bd35427dcbfad7e4377cf8083afc0acdc Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Fri, 2 Jul 2021 15:44:42 -0500 Subject: [PATCH 013/106] Rename CompletableOutputStream.isCompletelyWritten() to isComplete() and setCompletelyWritten() to setComplete() --- .../cantaloupe/cache/AzureStorageCache.java | 4 +-- .../cache/CompletableOutputStream.java | 10 +++---- .../cantaloupe/cache/DerivativeCache.java | 2 +- .../cantaloupe/cache/FilesystemCache.java | 4 +-- .../library/cantaloupe/cache/HeapCache.java | 2 +- .../library/cantaloupe/cache/JdbcCache.java | 4 +-- .../library/cantaloupe/cache/RedisCache.java | 2 +- .../library/cantaloupe/cache/S3Cache.java | 2 +- .../resource/ImageRepresentation.java | 2 +- .../cantaloupe/cache/AbstractCacheTest.java | 26 +++++++++---------- .../cantaloupe/cache/CacheFacadeTest.java | 14 +++++----- .../cantaloupe/cache/HeapCacheTest.java | 10 +++---- .../cantaloupe/cache/JdbcCacheTest.java | 6 ++--- .../library/cantaloupe/cache/S3CacheTest.java | 2 +- .../resource/ImageRequestHandlerTest.java | 4 +-- 15 files changed, 47 insertions(+), 47 deletions(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java index b6cf5082f..1ba50b610 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java @@ -67,7 +67,7 @@ private static class CustomBlobOutputStream /** * Constructor for an instance that writes into the given temporary - * blob. Upon closure, if the stream is {@link #isCompletelyWritten() + * blob. Upon closure, if the stream is {@link #isComplete() * completely written}, the temporary blob is copied into place and * deleted. Otherwise, the temporary blob is deleted. * @@ -96,7 +96,7 @@ public void close() throws IOException { blobOutputStream.flush(); blobOutputStream.close(); if (container != null) { - if (isCompletelyWritten()) { + if (isComplete()) { // Copy the temporary blob into place. CloudBlockBlob destBlob = container.getBlockBlobReference(blobKey); diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/CompletableOutputStream.java b/src/main/java/edu/illinois/library/cantaloupe/cache/CompletableOutputStream.java index a0d9b3f98..1f2a0e70a 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/CompletableOutputStream.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/CompletableOutputStream.java @@ -10,14 +10,14 @@ */ public abstract class CompletableOutputStream extends OutputStream { - private boolean isCompletelyWritten; + private boolean isComplete; - public boolean isCompletelyWritten() { - return isCompletelyWritten; + public boolean isComplete() { + return isComplete; } - public void setCompletelyWritten(boolean isCompletelyWritten) { - this.isCompletelyWritten = isCompletelyWritten; + public void setComplete(boolean isCompletelyWritten) { + this.isComplete = isCompletelyWritten; } } diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java index 8ce1b7f6a..cd03641f8 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java @@ -57,7 +57,7 @@ InputStream newDerivativeImageInputStream(OperationList opList) * *

      The {@link CompletableOutputStream#close()} method of the returned * instance must check the return value of {@link - * CompletableOutputStream#isCompletelyWritten()} before committing data + * CompletableOutputStream#isComplete()} before committing data * to the cache. If it returns {@code false}, any written data should be * discarded.

      * diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java index 6fd9110e7..7ea973d5c 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java @@ -181,7 +181,7 @@ public void close() { // If the written file is complete, move it into place. // Otherwise, delete it. - if (isCompletelyWritten()) { + if (isComplete()) { CFOS_LOGGER.debug("close(): moving {} to {}", tempFile, destinationFile); Files.move(tempFile, destinationFile, @@ -634,7 +634,7 @@ identifier, sourceImageTempFile(identifier), // work with newDerivativeImageOutputStream(). But this method does not // need that extra functionality, so setting it as completely written // here makes it behave like an ordinary OutputStream. - os.setCompletelyWritten(true); + os.setComplete(true); return os; } diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/HeapCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/HeapCache.java index acb869843..f870565e0 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/HeapCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/HeapCache.java @@ -184,7 +184,7 @@ private class HeapCacheOutputStream extends CompletableOutputStream { @Override public void close() throws IOException { LOGGER.debug("Closing stream for {}", opList); - if (isCompletelyWritten()) { + if (isComplete()) { Key key = itemKey(opList); Item item = new Item(wrappedStream.toByteArray()); cache.put(key, item); diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java index 01d6fe637..f147f0700 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java @@ -48,7 +48,7 @@ class JdbcCache implements DerivativeCache { /** * Wraps a {@link Blob} OutputStream, for writing an image to a BLOB. * The constructor creates a transaction that is committed on close if the - * stream is {@link CompletableOutputStream#isCompletelyWritten() + * stream is {@link CompletableOutputStream#isComplete() * completely written}. */ private class ImageBlobOutputStream extends CompletableOutputStream { @@ -92,7 +92,7 @@ private class ImageBlobOutputStream extends CompletableOutputStream { public void close() throws IOException { LOGGER.debug("Closing stream for {}", ops); try { - if (isCompletelyWritten()) { + if (isComplete()) { statement.executeUpdate(); connection.commit(); } else { diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/RedisCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/RedisCache.java index bb0c2053e..6996fde1b 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/RedisCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/RedisCache.java @@ -156,7 +156,7 @@ private static class RedisOutputStream extends CompletableOutputStream { @Override public void close() throws IOException { try { - if (isCompletelyWritten()) { + if (isComplete()) { connection.async().hset(hashKey, valueKey, bufferStream.toByteArray()); } diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java index 258d53f2c..c69b60c57 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java @@ -116,7 +116,7 @@ public void close() throws IOException { try { bufferStream.close(); byte[] data = bufferStream.toByteArray(); - if (isCompletelyWritten()) { + if (isComplete()) { // At this point, the client has received all image data, // but it is still waiting for the connection to close. // Uploading in a separate thread will allow this to happen diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRepresentation.java b/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRepresentation.java index d05cd3fa8..46e6d627f 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRepresentation.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRepresentation.java @@ -129,7 +129,7 @@ public void write(OutputStream responseOS) throws IOException { "cache simultaneously"); copyOrProcess(teeOS); cacheOS.flush(); - cacheOS.setCompletelyWritten(true); + cacheOS.setComplete(true); } else { copyOrProcess(responseOS); } diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java index f1c35b330..586453de5 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java @@ -110,7 +110,7 @@ void testNewDerivativeImageInputStreamWithZeroTTL() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList)) { Files.copy(imageFile, os); - os.setCompletelyWritten(true); + os.setComplete(true); } // Wait for it to upload @@ -144,7 +144,7 @@ void testNewDerivativeImageInputStreamWithNonzeroTTL() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(fixture, os); - os.setCompletelyWritten(true); + os.setComplete(true); } // Wait for it to finish, hopefully. @@ -182,7 +182,7 @@ void testNewDerivativeImageInputStreamConcurrently() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.setComplete(true); } return null; }, () -> { @@ -216,7 +216,7 @@ void testNewDerivativeImageOutputStream() throws Exception { try (CompletableOutputStream outputStream = instance.newDerivativeImageOutputStream(ops)) { Files.copy(fixture, outputStream); - outputStream.setCompletelyWritten(true); + outputStream.setComplete(true); } // Wait for it to upload @@ -245,7 +245,7 @@ void testNewDerivativeImageOutputStreamDoesNotLeaveDetritusWhenStreamIsIncomplet try (CompletableOutputStream outputStream = instance.newDerivativeImageOutputStream(ops)) { Files.copy(fixture, outputStream); - outputStream.setCompletelyWritten(false); // the whole point of the test + outputStream.setComplete(false); // the whole point of the test } // Wait for it to upload @@ -292,7 +292,7 @@ void testPurge() throws Exception { instance.newDerivativeImageOutputStream(opList)) { Path fixture = TestUtil.getImage(IMAGE); Files.copy(fixture, outputStream); - outputStream.setCompletelyWritten(true); + outputStream.setComplete(true); } // add the info @@ -332,7 +332,7 @@ void testPurgeWithIdentifier() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList1)) { Files.copy(fixture, os); - os.setCompletelyWritten(true); + os.setComplete(true); } instance.put(id1, new Info()); @@ -345,7 +345,7 @@ void testPurgeWithIdentifier() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList2)) { Files.copy(fixture, os); - os.setCompletelyWritten(true); + os.setComplete(true); } instance.put(id2, new Info()); @@ -385,7 +385,7 @@ void testPurgeWithOperationList() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops1)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.setComplete(true); } // Seed another derivative image @@ -396,7 +396,7 @@ void testPurgeWithOperationList() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops2)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.setComplete(true); } Thread.sleep(ASYNC_WAIT); @@ -436,7 +436,7 @@ void testPurgeInfos() throws Exception { instance.newDerivativeImageOutputStream(opList)) { Path fixture = TestUtil.getImage(IMAGE); Files.copy(fixture, outputStream); - outputStream.setCompletelyWritten(true); + outputStream.setComplete(true); } // add the info @@ -476,7 +476,7 @@ void testPurgeInvalid() throws Exception { try (CompletableOutputStream outputStream = instance.newDerivativeImageOutputStream(ops1)) { Files.copy(fixture, outputStream); - outputStream.setCompletelyWritten(true); + outputStream.setComplete(true); } // add an Info @@ -499,7 +499,7 @@ void testPurgeInvalid() throws Exception { try (CompletableOutputStream outputStream = instance.newDerivativeImageOutputStream(ops2)) { Files.copy(fixture2, outputStream); - outputStream.setCompletelyWritten(true); + outputStream.setComplete(true); } // add another info diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/CacheFacadeTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/CacheFacadeTest.java index d1350362e..e9111a778 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/CacheFacadeTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/CacheFacadeTest.java @@ -189,7 +189,7 @@ void testNewDerivativeImageInputStreamWhenDerivativeCacheIsEnabled() try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList)) { Files.copy(TestUtil.getImage("jpg"), os); - os.setCompletelyWritten(true); + os.setComplete(true); } try (InputStream is = instance.newDerivativeImageInputStream(opList)) { @@ -217,7 +217,7 @@ void testNewDerivativeImageOutputStreamWhenDerivativeCacheIsEnabled() try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList)) { assertNotNull(os); - os.setCompletelyWritten(true); + os.setComplete(true); } } @@ -253,7 +253,7 @@ void testPurge() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage("jpg"), os); - os.setCompletelyWritten(true); + os.setComplete(true); } // Add info to the derivative cache. @@ -299,7 +299,7 @@ void testPurgeWithIdentifier() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage("jpg"), os); - os.setCompletelyWritten(true); + os.setComplete(true); } // Add info to the derivative cache. @@ -343,7 +343,7 @@ void testPurgeAsyncWithIdentifier() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage("jpg"), os); - os.setCompletelyWritten(true); + os.setComplete(true); } // Add info to the derivative cache. @@ -378,7 +378,7 @@ void testPurgeWithOperationList() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(opList)) { Files.copy(TestUtil.getImage("jpg"), os); - os.setCompletelyWritten(true); + os.setComplete(true); } try (InputStream is = instance.newDerivativeImageInputStream(opList)) { @@ -447,7 +447,7 @@ void testPurgeInvalid() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage("jpg"), os); - os.setCompletelyWritten(true); + os.setComplete(true); } // Add info to the derivative cache. diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/HeapCacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/HeapCacheTest.java index 42a73409d..cb34460c9 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/HeapCacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/HeapCacheTest.java @@ -98,7 +98,7 @@ void testDumpToPersistentStore() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.setComplete(true); } instance.dumpToPersistentStore(); @@ -127,7 +127,7 @@ void testGetByteSize() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.setComplete(true); } assertEquals(5439, instance.getByteSize()); @@ -231,7 +231,7 @@ void testLoadFromPersistentStore() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.setComplete(true); } instance.dumpToPersistentStore(); @@ -296,7 +296,7 @@ void testPurgeExcessWithExcess() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.setComplete(true); } assertEquals(5439, instance.getByteSize()); @@ -317,7 +317,7 @@ void testPurgeExcessWithNoExcess() throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.setComplete(true); } long size = instance.getByteSize(); diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/JdbcCacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/JdbcCacheTest.java index edbfdb998..83895cb70 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/JdbcCacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/JdbcCacheTest.java @@ -106,7 +106,7 @@ private void seed(Connection connection) throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.setComplete(true); } Crop crop = new CropByPixels(50, 50, 50, 50); @@ -119,7 +119,7 @@ private void seed(Connection connection) throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.setComplete(true); } crop = new CropByPixels(10, 20, 50, 90); @@ -132,7 +132,7 @@ private void seed(Connection connection) throws Exception { try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(TestUtil.getImage(IMAGE), os); - os.setCompletelyWritten(true); + os.setComplete(true); } // persist some infos corresponding to the above images diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java index 80920af13..963567351 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java @@ -242,7 +242,7 @@ void testNewDerivativeImageInputStreamUpdatesLastModifiedTime() try (CompletableOutputStream os = instance.newDerivativeImageOutputStream(ops)) { Files.copy(fixture, os); - os.setCompletelyWritten(true); + os.setComplete(true); } // Wait for it to finish, hopefully. diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandlerTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandlerTest.java index 641f9e518..e2c633317 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandlerTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandlerTest.java @@ -218,7 +218,7 @@ void testHandleCallsCacheStreamingCallback() throws Exception { try (CompletableOutputStream os = cache.newDerivativeImageOutputStream(opList)) { os.write(new byte[] { 0x35, 0x35, 0x35 }); - os.setCompletelyWritten(true); + os.setComplete(true); } final IntrospectiveCallback callback = new IntrospectiveCallback(); @@ -348,7 +348,7 @@ void testHandleStreamsFromDerivativeCache() throws Exception { try (CompletableOutputStream os = cache.newDerivativeImageOutputStream(opList)) { os.write(expected); - os.setCompletelyWritten(true); + os.setComplete(true); } final IntrospectiveCallback callback = new IntrospectiveCallback(); From 1d33e8052d55e19fe9351e76dc442390c3fdfdd5 Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Mon, 23 Aug 2021 16:02:37 -0500 Subject: [PATCH 014/106] Add a processor.purge_incompatible_from_source_cache config key (#529) --- CHANGES.md | 4 +- UPGRADING.md | 1 + cantaloupe.properties.sample | 4 + .../library/cantaloupe/config/Key.java | 1 + .../resource/ImageRequestHandler.java | 19 ++ src/main/resources/admin.vm | 20 ++ .../resource/ImageRequestHandlerTest.java | 196 +++++++++++++++--- .../resource/admin/AdminResourceUITest.java | 2 + 8 files changed, 215 insertions(+), 32 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 77a8ef6ac..99a56d9fc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,11 +2,13 @@ # 6.0 -* The health endpoint is en/disabled via `endpoint.health.enabled` rather than +* The health endpoint is enabled via `endpoint.health.enabled` rather than `endpoint.api.enabled`. * HttpSource can be configured to send a ranged GET request instead of a HEAD request, enabling it to work with pre-signed URLs that do not allow HEAD requests. +* Added a configuration option to automatically purge source-cached images + whose format cannot be inferred. * Added an HTTP API method to purge all infos from the derivative cache. * The delegate script pathname can be set using the `-Dcantaloupe.delegate_script` VM argument, which takes precedence over the diff --git a/UPGRADING.md b/UPGRADING.md index 587efd51c..fd790a612 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -8,6 +8,7 @@ current version. 1. Add the following keys from the sample configuration: * `endpoint.health.enabled` * `HttpSource.BasicLookupStrategy.send_head_requests` + * `processor.purge_incompatible_from_source_cache` ## 4.1.x → 5.0 diff --git a/cantaloupe.properties.sample b/cantaloupe.properties.sample index e08bdd414..62a9a880e 100644 --- a/cantaloupe.properties.sample +++ b/cantaloupe.properties.sample @@ -370,6 +370,10 @@ processor.stream_retrieval_strategy = StreamStrategy # * `AbortStrategy` causes the request to fail. processor.fallback_retrieval_strategy = DownloadStrategy +# If true, images stored in the source cache for which no format can be +# inferred will be purged. +processor.purge_incompatible_from_source_cache = false + # Resolution of vector rasterization (of e.g. PDFs) at a scale of 1. processor.dpi = 150 diff --git a/src/main/java/edu/illinois/library/cantaloupe/config/Key.java b/src/main/java/edu/illinois/library/cantaloupe/config/Key.java index 2de3667ef..977da9d1c 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/config/Key.java +++ b/src/main/java/edu/illinois/library/cantaloupe/config/Key.java @@ -172,6 +172,7 @@ public enum Key { PROCESSOR_FALLBACK_RETRIEVAL_STRATEGY("processor.fallback_retrieval_strategy"), PROCESSOR_JPG_PROGRESSIVE("processor.jpg.progressive"), PROCESSOR_JPG_QUALITY("processor.jpg.quality"), + PROCESSOR_PURGE_INCOMPATIBLE_FROM_SOURCE_CACHE("processor.purge_incompatible_from_source_cache"), PROCESSOR_SELECTION_STRATEGY("processor.selection_strategy"), PROCESSOR_SHARPEN("processor.sharpen"), PROCESSOR_STREAM_RETRIEVAL_STRATEGY("processor.stream_retrieval_strategy"), diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandler.java b/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandler.java index 16d5142da..58af26d71 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandler.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandler.java @@ -1,5 +1,6 @@ package edu.illinois.library.cantaloupe.resource; +import edu.illinois.library.cantaloupe.async.TaskQueue; import edu.illinois.library.cantaloupe.cache.CacheFacade; import edu.illinois.library.cantaloupe.config.Configuration; import edu.illinois.library.cantaloupe.config.Key; @@ -23,6 +24,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @@ -414,6 +416,23 @@ public void handle(OutputStream outputStream) throws Exception { format, identifier); } } + if (config.getBoolean(Key.PROCESSOR_PURGE_INCOMPATIBLE_FROM_SOURCE_CACHE, false)) { + TaskQueue.getInstance().submit(() -> { + try { + cacheFacade.getSourceCacheFile(identifier).ifPresent(file -> { + try { + getLogger().debug("Deleting {}", file); + Files.delete(file); + } catch (IOException e) { + getLogger().warn("Failed to delete file from source cache: {}", + e.getMessage()); + } + }); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } throw new SourceFormatException(); } diff --git a/src/main/resources/admin.vm b/src/main/resources/admin.vm index 2134b10dc..45a46327e 100644 --- a/src/main/resources/admin.vm +++ b/src/main/resources/admin.vm @@ -2235,6 +2235,26 @@
      + ? + +
      + +
      +
      DPI ImageRequestHandler.builder().build()); } @Test - void testOptionallyWithDelegateProxyWithNonNullArguments() { + void optionallyWithDelegateProxyWithNonNullArguments() { DelegateProxy proxy = TestUtil.newDelegateProxy(); ImageRequestHandler handler = ImageRequestHandler.builder() .optionallyWithDelegateProxy(proxy, proxy.getRequestContext()) @@ -44,7 +50,7 @@ void testOptionallyWithDelegateProxyWithNonNullArguments() { } @Test - void testOptionallyWithDelegateProxyWithNullDelegateProxy() { + void optionallyWithDelegateProxyWithNullDelegateProxy() { RequestContext context = new RequestContext(); ImageRequestHandler handler = ImageRequestHandler.builder() .optionallyWithDelegateProxy(null, context) @@ -56,7 +62,7 @@ void testOptionallyWithDelegateProxyWithNullDelegateProxy() { } @Test - void testOptionallyWithDelegateProxyWithNullRequestContext() { + void optionallyWithDelegateProxyWithNullRequestContext() { DelegateProxy delegateProxy = TestUtil.newDelegateProxy(); ImageRequestHandler handler = ImageRequestHandler.builder() .withOperationList(new OperationList()) @@ -68,7 +74,7 @@ void testOptionallyWithDelegateProxyWithNullRequestContext() { } @Test - void testOptionallyWithDelegateProxyWithNullArguments() { + void optionallyWithDelegateProxyWithNullArguments() { ImageRequestHandler handler = ImageRequestHandler.builder() .withOperationList(new OperationList()) .optionallyWithDelegateProxy(null, null) @@ -79,14 +85,14 @@ void testOptionallyWithDelegateProxyWithNullArguments() { } @Test - void testWithDelegateProxyWithNullDelegateProxy() { + void withDelegateProxyWithNullDelegateProxy() { assertThrows(IllegalArgumentException.class, () -> ImageRequestHandler.builder() .withDelegateProxy(null, new RequestContext())); } @Test - void testWithDelegateProxyWithNullRequestContext() { + void withDelegateProxyWithNullRequestContext() { DelegateProxy delegateProxy = TestUtil.newDelegateProxy(); assertThrows(IllegalArgumentException.class, () -> ImageRequestHandler.builder() @@ -129,13 +135,13 @@ public void willProcessImage(Processor processor, Info info) { } @Test - void testHandleCallsPreAuthorizationCallback() throws Exception { + void handleCallsPreAuthorizationCallback() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.CACHE_SERVER_RESOLVE_FIRST, false); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); } // Configure the request. @@ -155,13 +161,13 @@ void testHandleCallsPreAuthorizationCallback() throws Exception { } @Test - void testHandleCallsAuthorizationCallback() throws Exception { + void handleCallsAuthorizationCallback() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.CACHE_SERVER_RESOLVE_FIRST, false); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); } // Configure the request. @@ -181,13 +187,13 @@ void testHandleCallsAuthorizationCallback() throws Exception { } @Test - void testHandleCallsCacheStreamingCallback() throws Exception { + void handleCallsCacheStreamingCallback() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.CACHE_SERVER_RESOLVE_FIRST, false); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); config.setProperty(Key.DERIVATIVE_CACHE_ENABLED, true); config.setProperty(Key.DERIVATIVE_CACHE, "HeapCache"); } @@ -233,13 +239,13 @@ void testHandleCallsCacheStreamingCallback() throws Exception { } @Test - void testHandleCallsInfoAvailableCallback() throws Exception { + void handleCallsInfoAvailableCallback() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.CACHE_SERVER_RESOLVE_FIRST, false); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); } // Configure the request. @@ -259,13 +265,13 @@ void testHandleCallsInfoAvailableCallback() throws Exception { } @Test - void testHandleCallsProcessingCallback() throws Exception { + void handleCallsProcessingCallback() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.CACHE_SERVER_RESOLVE_FIRST, false); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); } // Configure the request. @@ -285,12 +291,12 @@ void testHandleCallsProcessingCallback() throws Exception { } @Test - void testHandleProcessesImage() throws Exception { + void handleProcessesImage() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); } // Configure the request. @@ -310,13 +316,13 @@ void testHandleProcessesImage() throws Exception { } @Test - void testHandleStreamsFromDerivativeCache() throws Exception { + void handleStreamsFromDerivativeCache() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.CACHE_SERVER_RESOLVE_FIRST, false); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); config.setProperty(Key.DERIVATIVE_CACHE_ENABLED, true); config.setProperty(Key.DERIVATIVE_CACHE, "HeapCache"); } @@ -363,12 +369,12 @@ void testHandleStreamsFromDerivativeCache() throws Exception { } @Test - void testHandleWithFailedPreAuthorization() throws Exception { + void handleWithFailedPreAuthorization() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); } // Configure the request. @@ -405,12 +411,12 @@ public void willProcessImage(Processor processor, Info info) { } @Test - void testHandleWithFailedAuthorization() throws Exception { + void handleWithFailedAuthorization() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); } // Configure the request. @@ -447,12 +453,12 @@ public void willProcessImage(Processor processor, Info info) { } @Test - void testHandleWithIllegalPageIndex() throws Exception { + void handleWithIllegalPageIndex() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); } // Configure the request. @@ -474,12 +480,12 @@ void testHandleWithIllegalPageIndex() throws Exception { } @Test - void testHandleWithInvalidOperationList() throws Exception { + void handleWithInvalidOperationList() throws Exception { { // Configure the application. final Configuration config = Configuration.getInstance(); config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, - TestUtil.getImagesPath().toString() + "/"); + TestUtil.getImagesPath() + "/"); } // Configure the request. @@ -497,4 +503,132 @@ void testHandleWithInvalidOperationList() throws Exception { } } + @Test + void handleDeletesIncompatibleSourceCachedImageWhenSoConfigured() + throws Exception { + final WebServer server = new WebServer(); + try { + server.start(); + + { // Configure the application. + final Configuration config = Configuration.getInstance(); + config.setProperty(Key.SOURCE_STATIC, "HttpSource"); + config.setProperty(Key.HTTPSOURCE_URL_PREFIX, + server.getHTTPURI().toString() + "/"); + config.setProperty(Key.PROCESSOR_FALLBACK, + edu.illinois.library.cantaloupe.processor.MockStreamProcessor.class.getName()); + config.setProperty(Key.PROCESSOR_STREAM_RETRIEVAL_STRATEGY, + "CacheStrategy"); + config.setProperty(Key.PROCESSOR_PURGE_INCOMPATIBLE_FROM_SOURCE_CACHE, + true); // what this test is testing + config.setProperty(Key.SOURCE_CACHE, "FilesystemCache"); + config.setProperty(Key.SOURCE_CACHE_TTL, 300); + config.setProperty(Key.FILESYSTEMCACHE_PATHNAME, + Application.getTempPath().toString()); + } + + // Configure the request. + final OperationList opList = new OperationList(); + final Identifier identifier = new Identifier("jpg-rgb-64x48x8.jpg"); + final Metadata metadata = new Metadata(); + opList.setIdentifier(identifier); + Encode encode = new Encode(Format.get("jpg")); + encode.setCompression(Compression.JPEG); + encode.setQuality(80); + encode.setMetadata(metadata); + opList.add(encode); + + final CacheFacade cacheFacade = new CacheFacade(); + + try (ImageRequestHandler handler = ImageRequestHandler.builder() + .withOperationList(opList) + .build(); + OutputStream outputStream = OutputStream.nullOutputStream()) { + // The first request should cause the source image to be + // source-cached... + handler.handle(outputStream); + // Overwrite the source-cached image with garbage, destroying + // any format-signifying magic bytes. + Path file = cacheFacade.getSourceCacheFile(identifier).get(); + Files.write(file, "This is garbage".getBytes(StandardCharsets.UTF_8)); + // Send the same request again. The source cache will be + // consulted instead of the source. + handler.handle(outputStream); + fail("Expected a SourceFormatException"); + } catch (SourceFormatException e) { + // The delete happens asynchronously, so give it some time. + Thread.sleep(2000); + assertFalse(cacheFacade.getSourceCacheFile(identifier).isPresent()); + } + } finally { + server.stop(); + } + } + + @Test + void handleDoesNotDeleteIncompatibleSourceCachedImageWhenNotConfiguredTo() + throws Exception { + final WebServer server = new WebServer(); + try { + server.start(); + + { // Configure the application. + final Configuration config = Configuration.getInstance(); + config.setProperty(Key.SOURCE_STATIC, "HttpSource"); + config.setProperty(Key.HTTPSOURCE_URL_PREFIX, + server.getHTTPURI().toString() + "/"); + config.setProperty(Key.PROCESSOR_FALLBACK, + edu.illinois.library.cantaloupe.processor.MockStreamProcessor.class.getName()); + config.setProperty(Key.PROCESSOR_STREAM_RETRIEVAL_STRATEGY, + "CacheStrategy"); + config.setProperty(Key.PROCESSOR_PURGE_INCOMPATIBLE_FROM_SOURCE_CACHE, + false); // what this test is testing + config.setProperty(Key.SOURCE_CACHE, "FilesystemCache"); + config.setProperty(Key.SOURCE_CACHE_TTL, 300); + config.setProperty(Key.FILESYSTEMCACHE_PATHNAME, + Application.getTempPath().toString()); + } + + // Configure the request. + final OperationList opList = new OperationList(); + final Identifier identifier = new Identifier("jpg-rgb-64x48x8.jpg"); + final Metadata metadata = new Metadata(); + opList.setIdentifier(identifier); + Encode encode = new Encode(Format.get("jpg")); + encode.setCompression(Compression.JPEG); + encode.setQuality(80); + encode.setMetadata(metadata); + opList.add(encode); + + final CacheFacade cacheFacade = new CacheFacade(); + + try (ImageRequestHandler handler = ImageRequestHandler.builder() + .withOperationList(opList) + .build(); + OutputStream outputStream = OutputStream.nullOutputStream()) { + // The first request should cause the source image to be + // source-cached... + handler.handle(outputStream); + // Overwrite the source-cached image with garbage, destroying + // any format-signifying magic bytes. + Path file = cacheFacade.getSourceCacheFile(identifier).get(); + Files.write(file, "This is garbage".getBytes(StandardCharsets.UTF_8)); + // Send the same request again. The source cache will be + // consulted instead of the source. + handler.handle(outputStream); + fail("Expected a SourceFormatException"); + } catch (SourceFormatException e) { + // The source-cached file is not supposed to get deleted, but + // it will happen asynchronously if it does, so give it some + // time. + Thread.sleep(2000); + Path file = cacheFacade.getSourceCacheFile(identifier).get(); + assertTrue(Files.exists(file)); + Files.delete(file); + } + } finally { + server.stop(); + } + } + } diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/admin/AdminResourceUITest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/admin/AdminResourceUITest.java index 462204140..7847b0bf0 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/admin/AdminResourceUITest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/admin/AdminResourceUITest.java @@ -497,6 +497,7 @@ void testProcessorsSection() throws Exception { selectByValue("StreamStrategy"); selectNamed(Key.PROCESSOR_FALLBACK_RETRIEVAL_STRATEGY). selectByValue("CacheStrategy"); + inputNamed(Key.PROCESSOR_PURGE_INCOMPATIBLE_FROM_SOURCE_CACHE).click(); inputNamed(Key.PROCESSOR_DPI).sendKeys("300"); selectNamed(Key.PROCESSOR_BACKGROUND_COLOR).selectByValue("white"); selectNamed(Key.PROCESSOR_UPSCALE_FILTER). @@ -542,6 +543,7 @@ void testProcessorsSection() throws Exception { config.getString(Key.PROCESSOR_STREAM_RETRIEVAL_STRATEGY)); assertEquals("CacheStrategy", config.getString(Key.PROCESSOR_FALLBACK_RETRIEVAL_STRATEGY)); + assertTrue(config.getBoolean(Key.PROCESSOR_PURGE_INCOMPATIBLE_FROM_SOURCE_CACHE)); assertEquals(300, config.getInt(Key.PROCESSOR_DPI)); assertEquals("white", config.getString(Key.PROCESSOR_BACKGROUND_COLOR)); assertEquals("triangle", From 7556bcaafb168a9c9fb8b0458196bd6ebdb3128a Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Mon, 23 Aug 2021 16:48:24 -0500 Subject: [PATCH 015/106] Add --illegal-access=permit argument to the surefire plugin to support JAI in JDK 16 --- pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pom.xml b/pom.xml index 70ced69d2..1efed2992 100644 --- a/pom.xml +++ b/pom.xml @@ -419,6 +419,8 @@ random false + + --illegal-access=permit From 27bb281ba3bac92a24af678b80e7d0e2956fef87 Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Mon, 23 Aug 2021 16:48:45 -0500 Subject: [PATCH 016/106] Catch a spurious NoSuchFileException in tearDown() --- .../illinois/library/cantaloupe/cache/FilesystemCacheTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/FilesystemCacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/FilesystemCacheTest.java index 747efba7e..39e9abffb 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/FilesystemCacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/FilesystemCacheTest.java @@ -27,6 +27,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryNotEmptyException; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; @@ -59,7 +60,7 @@ public void setUp() throws Exception { public void tearDown() throws IOException { try { Files.walkFileTree(fixturePath, new DeletingFileVisitor()); - } catch (DirectoryNotEmptyException e) { + } catch (NoSuchFileException | DirectoryNotEmptyException e) { // This happens in Windows 7 (maybe other versions?) sometimes; not // sure why, but it shouldn't result in a test failure. System.err.println(e.getMessage()); From 3cf62b8c5f7b25621b2f896186d20954d3b9ab72 Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Mon, 30 Aug 2021 11:53:57 -0500 Subject: [PATCH 017/106] Include *iml & *.ipr, exclude *.iws --- .gitignore | 4 +- cantaloupe.iml | 2032 ++++++++++++++++++++++++++++++++++++++++++++++++ cantaloupe.ipr | 107 +++ 3 files changed, 2141 insertions(+), 2 deletions(-) create mode 100644 cantaloupe.iml create mode 100644 cantaloupe.ipr diff --git a/.gitignore b/.gitignore index 6ec6b4cad..3effafb05 100644 --- a/.gitignore +++ b/.gitignore @@ -8,8 +8,8 @@ Thumbs.db .settings/* # IntelliJ stuff -.idea/* -*.iml +/.idea +/*.iws # NetBeans stuff nbactions.xml diff --git a/cantaloupe.iml b/cantaloupe.iml new file mode 100644 index 000000000..e98a7ed48 --- /dev/null +++ b/cantaloupe.iml @@ -0,0 +1,2032 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/cantaloupe.ipr b/cantaloupe.ipr new file mode 100644 index 000000000..5850a126f --- /dev/null +++ b/cantaloupe.ipr @@ -0,0 +1,107 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 2daceccb9204921b9bd46df8a8734733403c4d02 Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Sat, 11 Dec 2021 20:34:36 -0600 Subject: [PATCH 018/106] S3Source supports multiple endpoints when using ScriptLookupStrategy --- CHANGES.md | 1 + delegates.rb.sample | 7 +- .../source/S3HTTPImageInputStreamClient.java | 2 +- .../cantaloupe/source/S3ObjectInfo.java | 71 +++++++++++++++++-- .../library/cantaloupe/source/S3Source.java | 68 +++++++++++++----- .../S3HTTPImageInputStreamClientTest.java | 4 +- .../cantaloupe/source/S3SourceTest.java | 56 +++++++++++---- .../source/S3StreamFactoryTest.java | 4 +- 8 files changed, 170 insertions(+), 43 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c7bf37ac7..7d8274fdd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,7 @@ * HttpSource can be configured to send a ranged GET request instead of a HEAD request, enabling it to work with pre-signed URLs that do not allow HEAD requests. +* S3Source supports multiple endpoints when using ScriptLookupStrategy. * Added a configuration option to automatically purge source-cached images whose format cannot be inferred. * Added an HTTP API method to purge all infos from the derivative cache. diff --git a/delegates.rb.sample b/delegates.rb.sample index 728096743..78f720721 100644 --- a/delegates.rb.sample +++ b/delegates.rb.sample @@ -280,8 +280,11 @@ class CustomDelegate # should be used instead. # # @param options [Hash] Empty hash. - # @return [Hash,nil] Hash containing `bucket` and `key` keys; - # or nil if not found. + # @return [Hash,nil] Hash containing `bucket` and `key` keys. + # It may also contain an `endpoint` key, indicating that the endpoint + # is different from the one set in the configuration. In that case, + # it may also contain `region`, `access_key_id`, and/or + # `secret_access_key` keys. # def s3source_object_info(options = {}) end diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/S3HTTPImageInputStreamClient.java b/src/main/java/edu/illinois/library/cantaloupe/source/S3HTTPImageInputStreamClient.java index 6c9e49d62..3d06f7807 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/S3HTTPImageInputStreamClient.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/S3HTTPImageInputStreamClient.java @@ -36,7 +36,7 @@ class S3HTTPImageInputStreamClient implements HTTPImageInputStreamClient { @Override public Response sendHEADRequest() throws IOException { - final S3Client client = S3Source.getClientInstance(); + final S3Client client = S3Source.getClientInstance(objectInfo); final String bucket = objectInfo.getBucketName(); final String key = objectInfo.getKey(); try { diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/S3ObjectInfo.java b/src/main/java/edu/illinois/library/cantaloupe/source/S3ObjectInfo.java index cfaa8afe6..bf11539ca 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/S3ObjectInfo.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/S3ObjectInfo.java @@ -1,19 +1,32 @@ package edu.illinois.library.cantaloupe.source; +/** + * Contains information needed to access an object in S3. + */ final class S3ObjectInfo { - private String bucketName, key; + private String region, endpoint, accessKeyID, secretAccessKey, bucketName, + key; private long length = -1; - S3ObjectInfo(String key, String bucketName) { - this.key = key; - this.bucketName = bucketName; + /** + * @return Access key ID. May be {@code null}. + */ + String getAccessKeyID() { + return accessKeyID; } String getBucketName() { return bucketName; } + /** + * @return Service endpoint URI. May be {@code null}. + */ + String getEndpoint() { + return endpoint; + } + String getKey() { return key; } @@ -22,13 +35,61 @@ long getLength() { return length; } + /** + * @return Endpoint AWS region. Only used by AWS endpoints. May be {@code + * null}. + */ + String getRegion() { + return region; + } + + /** + * @return Secret access key. May be {@code null}. + */ + String getSecretAccessKey() { + return secretAccessKey; + } + + void setAccessKeyID(String accessKeyID) { + this.accessKeyID = accessKeyID; + } + + void setBucketName(String bucketName) { + this.bucketName = bucketName; + } + + void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } + + void setKey(String key) { + this.key = key; + } + void setLength(long length) { this.length = length; } + void setRegion(String region) { + this.region = region; + } + + void setSecretAccessKey(String secretAccessKey) { + this.secretAccessKey = secretAccessKey; + } + @Override public String toString() { - return getBucketName() + "/" + getKey(); + StringBuilder b = new StringBuilder(); + b.append("[endpoint: ").append(getEndpoint()).append("] "); + b.append("[region: ").append(getRegion()).append("] "); + String tmp = getAccessKeyID() != null ? "******" : "null"; + b.append("[accessKeyID: ").append(tmp).append("] "); + tmp = getSecretAccessKey() != null ? "******" : "null"; + b.append("[secretAccessKey: ").append(tmp).append("] "); + b.append("[bucket: ").append(getBucketName()).append("] "); + b.append("[key: ").append(getKey()).append("]"); + return b.toString(); } } diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/S3Source.java b/src/main/java/edu/illinois/library/cantaloupe/source/S3Source.java index 6b18d98a0..ac60a4395 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/S3Source.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/S3Source.java @@ -28,6 +28,7 @@ import java.net.URISyntaxException; import java.nio.file.AccessDeniedException; import java.nio.file.NoSuchFileException; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -79,8 +80,6 @@ * * * @author Alex Dolski UIUC - * @see - * Minio Java Client API Reference */ final class S3Source extends AbstractSource implements Source { @@ -189,11 +188,16 @@ public T next() { LoggerFactory.getLogger(S3Source.class); /** - * Range used to infer the source image format. + * Byte range used to infer the source image format. */ private static final Range FORMAT_INFERENCE_RANGE = new Range(0, 32); - private static S3Client client; + /** + * The keys are endpoint URIs. The default client's key is {@code null}. + * This is not thread-safe, so should only be accessed via {@link + * #getClientInstance(S3ObjectInfo)}. + */ + private static final Map CLIENTS = new HashMap<>(); /** * Cached by {@link #getObjectInfo()}. @@ -207,25 +211,44 @@ public T next() { private FormatIterator formatIterator = new FormatIterator<>(); - static synchronized S3Client getClientInstance() { + static synchronized S3Client getClientInstance(S3ObjectInfo info) { + String endpoint = info.getEndpoint(); + S3Client client = CLIENTS.get(endpoint); if (client == null) { final Configuration config = Configuration.getInstance(); - final String endpointStr = config.getString(Key.S3SOURCE_ENDPOINT); + if (endpoint == null) { + endpoint = config.getString(Key.S3SOURCE_ENDPOINT); + } + // Convert the endpoint string into a URI which is required by the + // client builder. URI endpointURI = null; - if (endpointStr != null) { + if (endpoint != null) { try { - endpointURI = new URI(endpointStr); + endpointURI = new URI(endpoint); } catch (URISyntaxException e) { LOGGER.error("Invalid URI for {}: {}", Key.S3SOURCE_ENDPOINT, e.getMessage()); } } + String region = info.getRegion(); + if (region == null) { + region = config.getString(Key.S3SOURCE_REGION); + } + String accessKeyID = info.getAccessKeyID(); + if (accessKeyID == null) { + accessKeyID = config.getString(Key.S3SOURCE_ACCESS_KEY_ID); + } + String secretAccessKey = info.getSecretAccessKey(); + if (secretAccessKey == null) { + secretAccessKey = config.getString(Key.S3SOURCE_SECRET_KEY); + } client = new S3ClientBuilder() - .accessKeyID(config.getString(Key.S3SOURCE_ACCESS_KEY_ID)) - .secretAccessKey(config.getString(Key.S3SOURCE_SECRET_KEY)) + .accessKeyID(accessKeyID) + .secretAccessKey(secretAccessKey) .endpointURI(endpointURI) - .region(config.getString(Key.S3SOURCE_REGION)) + .region(region) .build(); + CLIENTS.put(endpoint, client); } return client; } @@ -248,7 +271,7 @@ static InputStream newObjectInputStream(S3ObjectInfo info) */ static InputStream newObjectInputStream(S3ObjectInfo info, Range range) throws IOException { - final S3Client client = getClientInstance(); + final S3Client client = getClientInstance(info); try { GetObjectRequest request; if (range != null) { @@ -290,7 +313,7 @@ private S3ObjectAttributes getObjectAttributes() throws IOException { final S3ObjectInfo info = getObjectInfo(); final String bucket = info.getBucketName(); final String key = info.getKey(); - final S3Client client = getClientInstance(); + final S3Client client = getClientInstance(info); try { HeadObjectResponse response = client.headObject(HeadObjectRequest.builder() .bucket(bucket) @@ -349,7 +372,10 @@ private S3ObjectInfo getObjectInfoUsingBasicStrategy() { final String keyPrefix = config.getString(Key.S3SOURCE_PATH_PREFIX, ""); final String keySuffix = config.getString(Key.S3SOURCE_PATH_SUFFIX, ""); final String key = keyPrefix + identifier.toString() + keySuffix; - return new S3ObjectInfo(key, bucketName); + S3ObjectInfo info = new S3ObjectInfo(); + info.setBucketName(bucketName); + info.setKey(key); + return info; } /** @@ -372,12 +398,18 @@ private S3ObjectInfo getObjectInfoUsingDelegateStrategy() } if (result.containsKey("bucket") && result.containsKey("key")) { - String bucketName = result.get("bucket"); - String objectKey = result.get("key"); - return new S3ObjectInfo(objectKey, bucketName); + final S3ObjectInfo info = new S3ObjectInfo(); + info.setBucketName(result.get("bucket")); + info.setKey(result.get("key")); + // These may be null. + info.setRegion(result.get("region")); + info.setEndpoint(result.get("endpoint")); + info.setAccessKeyID(result.get("access_key_id")); + info.setSecretAccessKey(result.get("secret_access_key")); + return info; } else { throw new IllegalArgumentException( - "Returned hash does not include bucket and key"); + "Returned hash must include bucket and key"); } } diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/S3HTTPImageInputStreamClientTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/S3HTTPImageInputStreamClientTest.java index 2579b17ab..0df700cb3 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/S3HTTPImageInputStreamClientTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/S3HTTPImageInputStreamClientTest.java @@ -103,7 +103,9 @@ public void setUp() throws Exception { configureS3Source(); seedFixtures(); - S3ObjectInfo info = new S3ObjectInfo(FIXTURE_KEY, bucket()); + S3ObjectInfo info = new S3ObjectInfo(); + info.setBucketName(bucket()); + info.setKey(FIXTURE_KEY); info.setLength(1584); instance = new S3HTTPImageInputStreamClient(info); diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/S3SourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/S3SourceTest.java index 63fc3b0cc..56d42aa35 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/S3SourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/S3SourceTest.java @@ -202,27 +202,55 @@ void useScriptLookupStrategy() { } } + /* getClientInstance() */ + + @Test + void getClientInstanceReturnsDefaultClient() { + assertNotNull(S3Source.getClientInstance(new S3ObjectInfo())); + } + + @Test + void getClientInstanceReturnsUniqueClientsPerEndpoint() { + S3ObjectInfo info1 = new S3ObjectInfo(); + S3ObjectInfo info2 = new S3ObjectInfo(); + info2.setEndpoint("http://example.org/endpoint"); + S3Client client1 = S3Source.getClientInstance(info1); + S3Client client2 = S3Source.getClientInstance(info2); + assertNotSame(client1, client2); + } + + @Test + void getClientInstanceCachesReturnedClients() { + S3ObjectInfo info1 = new S3ObjectInfo(); + S3ObjectInfo info2 = new S3ObjectInfo(); + info1.setEndpoint("http://example.org/endpoint"); + info2.setEndpoint(info1.getEndpoint()); + S3Client client1 = S3Source.getClientInstance(info1); + S3Client client2 = S3Source.getClientInstance(info2); + assertSame(client1, client2); + } + /* checkAccess() */ @Test - void testCheckAccessUsingBasicLookupStrategyWithPresentUnreadableImage() { + void checkAccessUsingBasicLookupStrategyWithPresentUnreadableImage() { // TODO: write this } @Test - void testCheckAccessUsingScriptLookupStrategyWithPresentReadableImage() + void checkAccessUsingScriptLookupStrategyWithPresentReadableImage() throws Exception { useScriptLookupStrategy(); instance.checkAccess(); } @Test - void testCheckAccessUsingScriptLookupStrategyWithPresentUnreadableImage() { + void checkAccessUsingScriptLookupStrategyWithPresentUnreadableImage() { // TODO: write this } @Test - void testCheckAccessUsingScriptLookupStrategyWithMissingImage() { + void checkAccessUsingScriptLookupStrategyWithMissingImage() { useScriptLookupStrategy(); Identifier identifier = new Identifier("bogus"); @@ -235,8 +263,7 @@ void testCheckAccessUsingScriptLookupStrategyWithMissingImage() { } @Test - void testCheckAccessUsingScriptLookupStrategyReturningHash() - throws Exception { + void checkAccessUsingScriptLookupStrategyReturningHash() throws Exception { useScriptLookupStrategy(); Identifier identifier = new Identifier("bucket:" + getBucket() + @@ -251,7 +278,7 @@ void testCheckAccessUsingScriptLookupStrategyReturningHash() } @Test - void testCheckAccessUsingScriptLookupStrategyWithMissingKeyInReturnedHash() { + void checkAccessUsingScriptLookupStrategyWithMissingKeyInReturnedHash() { useScriptLookupStrategy(); Identifier identifier = new Identifier("key:" + OBJECT_KEY_WITH_CONTENT_TYPE_AND_RECOGNIZED_EXTENSION); @@ -266,7 +293,7 @@ void testCheckAccessUsingScriptLookupStrategyWithMissingKeyInReturnedHash() { /* getFormatIterator() */ @Test - void testGetFormatIteratorHasNext() { + void getFormatIteratorHasNext() { S3Source source = newInstance(); source.setIdentifier(new Identifier(OBJECT_KEY_WITH_CONTENT_TYPE_AND_RECOGNIZED_EXTENSION)); S3Source.FormatIterator it = source.getFormatIterator(); @@ -283,7 +310,7 @@ void testGetFormatIteratorHasNext() { } @Test - void testGetFormatIteratorNext() { + void getFormatIteratorNext() { S3Source source = newInstance(); source.setIdentifier(new Identifier(OBJECT_KEY_WITH_NO_CONTENT_TYPE_AND_INCORRECT_EXTENSION)); @@ -298,12 +325,12 @@ void testGetFormatIteratorNext() { /* getObjectInfo() */ @Test - void testGetObjectInfo() throws Exception { + void getObjectInfo() throws Exception { assertNotNull(instance.getObjectInfo()); } @Test - void testGetObjectInfoUsingBasicLookupStrategyWithPrefixAndSuffix() + void getObjectInfoUsingBasicLookupStrategyWithPrefixAndSuffix() throws Exception { Configuration config = Configuration.getInstance(); config.setProperty(Key.S3SOURCE_PATH_PREFIX, "/prefix/"); @@ -314,7 +341,7 @@ void testGetObjectInfoUsingBasicLookupStrategyWithPrefixAndSuffix() } @Test - void testGetObjectInfoUsingBasicLookupStrategyWithoutPrefixOrSuffix() + void getObjectInfoUsingBasicLookupStrategyWithoutPrefixOrSuffix() throws Exception { Configuration config = Configuration.getInstance(); config.setProperty(Key.S3SOURCE_PATH_PREFIX, ""); @@ -327,13 +354,12 @@ void testGetObjectInfoUsingBasicLookupStrategyWithoutPrefixOrSuffix() /* newStreamFactory() */ @Test - void testNewStreamFactoryUsingBasicLookupStrategy() throws Exception { + void newStreamFactoryUsingBasicLookupStrategy() throws Exception { assertNotNull(instance.newStreamFactory()); } @Test - void testNewStreamFactoryUsingScriptLookupStrategy() - throws Exception { + void newStreamFactoryUsingScriptLookupStrategy() throws Exception { useScriptLookupStrategy(); assertNotNull(instance.newStreamFactory()); } diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/S3StreamFactoryTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/S3StreamFactoryTest.java index b6527a90f..c1432a948 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/S3StreamFactoryTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/S3StreamFactoryTest.java @@ -104,7 +104,9 @@ public void setUp() throws Exception { configureS3Source(); seedFixtures(); - S3ObjectInfo info = new S3ObjectInfo(FIXTURE_KEY, bucket()); + S3ObjectInfo info = new S3ObjectInfo(); + info.setBucketName(bucket()); + info.setKey(FIXTURE_KEY); info.setLength(1584); instance = new S3StreamFactory(info); From 7747d7cf66b6065c6af04cf3f825733760c43a50 Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Thu, 23 Dec 2021 15:44:33 -0600 Subject: [PATCH 019/106] Remove --- cantaloupe.ipr | 107 ------------------------------------------------- 1 file changed, 107 deletions(-) delete mode 100644 cantaloupe.ipr diff --git a/cantaloupe.ipr b/cantaloupe.ipr deleted file mode 100644 index 5850a126f..000000000 --- a/cantaloupe.ipr +++ /dev/null @@ -1,107 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From a15dfcd90ba8297d3b9510e21b57d8c376f65380 Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Thu, 23 Dec 2021 15:45:20 -0600 Subject: [PATCH 020/106] Avoid ArrayList resizing in chunkify() --- .../java/edu/illinois/library/cantaloupe/util/ArrayUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/util/ArrayUtils.java b/src/main/java/edu/illinois/library/cantaloupe/util/ArrayUtils.java index a751586bb..ce9b21600 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/util/ArrayUtils.java +++ b/src/main/java/edu/illinois/library/cantaloupe/util/ArrayUtils.java @@ -17,7 +17,8 @@ public final class ArrayUtils { * @return Chunked data. */ public static List chunkify(byte[] bytes, int maxChunkSize) { - final List chunks = new ArrayList<>(10); + final int listSize = (int) Math.ceil(bytes.length / (double) maxChunkSize); + final List chunks = new ArrayList<>(listSize); if (bytes.length <= maxChunkSize) { chunks.add(bytes); } else { From f3dddb09d8aabcc631f146438cace2c106cd6e6e Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Mon, 7 Mar 2022 09:23:07 -0600 Subject: [PATCH 021/106] S3Cache uses multipart uploads --- CHANGES.md | 2 + .../library/cantaloupe/cache/S3Cache.java | 187 ++-------- .../cache/S3MultipartAsyncOutputStream.java | 318 ++++++++++++++++++ .../S3MultipartAsyncOutputStreamTest.java | 311 +++++++++++++++++ 4 files changed, 656 insertions(+), 162 deletions(-) create mode 100644 src/main/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStream.java create mode 100644 src/test/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStreamTest.java diff --git a/CHANGES.md b/CHANGES.md index b4a9d2b1a..88b0d66bc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,8 @@ * The delegate script pathname can be set using the `-Dcantaloupe.delegate_script` VM argument, which takes precedence over the `delegate_script.pathname` configuration key. +* S3Cache uses multipart uploads, which reduces memory usage when caching + derivatives larger than 5 MB. ## 5.0.6 diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java index c5f49399c..b453a2850 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java @@ -30,10 +30,8 @@ import software.amazon.awssdk.services.s3.model.S3Object; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -64,156 +62,6 @@ */ class S3Cache implements DerivativeCache { - /** - *

      Wraps a {@link ByteArrayOutputStream} for upload to S3.

      - * - *

      N.B.: S3 does not allow uploads without a {@code Content-Length} - * header, which cannot be provided when streaming an unknown amount of - * data (which this class is going to be doing all the time). From the - * documentation of {@link PutObjectRequest}:

      - * - *
      "When uploading directly from an input stream, content - * length must be specified before data can be uploaded to Amazon S3. If - * not provided, the library will have to buffer the contents of the input - * stream in order to calculate it. Amazon S3 explicitly requires that the - * content length be sent in the request headers before any of the data is - * sent."
      - * - *

      Since it's not possible to write an {@link OutputStream} of unknown - * length to the S3 client as the {@link Cache} interface requires, this - * class buffers written data in a byte array before uploading it to S3 - * upon closure. (The upload is submitted to the - * {@link ThreadPool#getInstance() application thread pool} in order for - * {@link #close()} to be able to return immediately.)

      - */ - private static class S3OutputStream extends CompletableOutputStream { - - private final ByteArrayOutputStream bufferStream = - new ByteArrayOutputStream(); - private final S3Client client; - private final String bucketName; - private final String objectKey; - private final String contentType; - - /** - * @param client S3 client. - * @param bucketName S3 bucket name. - * @param objectKey S3 object key. - * @param contentType Media type. - */ - S3OutputStream(final S3Client client, - final String bucketName, - final String objectKey, - final String contentType) { - this.client = client; - this.bucketName = bucketName; - this.objectKey = objectKey; - this.contentType = contentType; - } - - @Override - public void close() throws IOException { - try { - bufferStream.close(); - byte[] data = bufferStream.toByteArray(); - if (isComplete()) { - // At this point, the client has received all image data, - // but it is still waiting for the connection to close. - // Uploading in a separate thread will allow this to happen - // immediately. - ThreadPool.getInstance().submit(new S3Upload( - client, data, bucketName, objectKey, - contentType, null)); - } - } finally { - super.close(); - } - } - - @Override - public void flush() throws IOException { - bufferStream.flush(); - } - - @Override - public void write(int b) { - bufferStream.write(b); - } - - @Override - public void write(byte[] b) throws IOException { - bufferStream.write(b); - } - - @Override - public void write(byte[] b, int off, int len) { - bufferStream.write(b, off, len); - } - - } - - private static class S3Upload implements Runnable { - - private static final Logger UPLOAD_LOGGER = - LoggerFactory.getLogger(S3Upload.class); - - private final String bucketName, contentEncoding, contentType, objectKey; - private final byte[] data; - private final S3Client client; - - /** - * @param client S3 client. - * @param data Data to upload. - * @param bucketName S3 bucket name. - * @param objectKey S3 object key. - * @param contentType Media type. - * @param contentEncoding Content encoding. May be {@code null}. - */ - S3Upload(S3Client client, - byte[] data, - String bucketName, - String objectKey, - String contentType, - String contentEncoding) { - this.client = client; - this.bucketName = bucketName; - this.data = data; - this.contentType = contentType; - this.contentEncoding = contentEncoding; - this.objectKey = objectKey; - } - - @Override - public void run() { - if (data.length > 0) { - PutObjectRequest request = PutObjectRequest.builder() - .bucket(bucketName) - .key(objectKey) - .contentType(contentType) - .contentEncoding(contentEncoding) - .build(); - final Stopwatch watch = new Stopwatch(); - - UPLOAD_LOGGER.debug("Uploading {} bytes to {} in bucket {}", - data.length, request.key(), request.bucket()); - - try (ByteArrayInputStream is = new ByteArrayInputStream(data)) { - client.putObject(request, - RequestBody.fromInputStream(is, data.length)); - } catch (IOException e) { - UPLOAD_LOGGER.warn(e.getMessage(), e); - } - - UPLOAD_LOGGER.trace("Wrote {} bytes to {} in bucket {} in {}", - data.length, request.key(), request.bucket(), - watch); - } else { - UPLOAD_LOGGER.trace("No data to upload; returning"); - } - } - - } - private static final Logger LOGGER = LoggerFactory.getLogger(S3Cache.class); @@ -345,10 +193,10 @@ public InputStream newDerivativeImageInputStream(OperationList opList) @Override public CompletableOutputStream newDerivativeImageOutputStream(OperationList opList) { - final String objectKey = getObjectKey(opList); - final String bucketName = getBucketName(); - final S3Client client = getClientInstance(); - return new S3OutputStream(client, bucketName, objectKey, + final String objectKey = getObjectKey(opList); + final String bucketName = getBucketName(); + final S3Client client = getClientInstance(); + return new S3MultipartAsyncOutputStream(client, bucketName, objectKey, opList.getOutputFormat().getPreferredMediaType().toString()); } @@ -528,13 +376,28 @@ public void put(Identifier identifier, Info info) throws IOException { @Override public void put(Identifier identifier, String info) throws IOException { LOGGER.debug("put(): caching info for {}", identifier); - final S3Client client = getClientInstance(); - final String objectKey = getObjectKey(identifier); - final String bucketName = getBucketName(); + final Stopwatch watch = new Stopwatch(); + + PutObjectRequest request = PutObjectRequest.builder() + .bucket(getBucketName()) + .key(getObjectKey(identifier)) + .contentType(MediaType.APPLICATION_JSON.toString()) + .contentEncoding("UTF-8") + .build(); + byte[] data = info.getBytes(StandardCharsets.UTF_8); + LOGGER.trace("put(): uploading {} bytes to {} in bucket {}", + data.length, request.key(), request.bucket()); + + try (ByteArrayInputStream is = new ByteArrayInputStream(data)) { + getClientInstance().putObject(request, + RequestBody.fromInputStream(is, data.length)); + } catch (IOException e) { + LOGGER.warn(e.getMessage(), e); + } - new S3Upload(client, info.getBytes(StandardCharsets.UTF_8), - bucketName, objectKey, MediaType.APPLICATION_JSON.toString(), - "UTF-8").run(); + LOGGER.trace("put(): wrote {} bytes to {} in bucket {} in {}", + data.length, request.key(), request.bucket(), + watch); } /** diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStream.java b/src/main/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStream.java new file mode 100644 index 000000000..00b93b54e --- /dev/null +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStream.java @@ -0,0 +1,318 @@ +package edu.illinois.library.cantaloupe.cache; + +import edu.illinois.library.cantaloupe.async.ThreadPool; +import org.apache.commons.compress.utils.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +/** + *

      Uploads written data to S3 in parts without blocking on uploads.

      + * + *

      The multi-part upload process involves three types of operations: + * creating the upload, uploading the parts, and completing the upload. Each of + * these are encapsulated in {@link Runnable runnable} inner classes. The + * {@link #write} methods add appropriate instances of these to a queue which + * is consumed by a worker running in the {@link ThreadPool#getInstance() + * application thread pool}.

      + * + *

      Clients will notice that calls to {@link #write} and {@link #close()} + * (that would otherwise block on communication with S3) return immediately. + * After {@link #close()} returns, the resulting object will take a little bit + * of time to appear in the bucket.

      + * + *

      Multi-part uploads can reduce memory usage when uploading objects larger + * than the part length, as that is roughly the maximum amount that has to be + * buffered in memory (provided that the length of the byte array passed to + * either of the {@link #write} methods is not greater than the part + * length).

      + * + * @author Alex Dolski UIUC + * @since 6.0 + */ +public class S3MultipartAsyncOutputStream extends CompletableOutputStream { + + private interface TerminalTask {} + + private static class Worker implements Runnable { + private final BlockingQueue workQueue = + new LinkedBlockingQueue<>(); + private boolean isDone, isStopped; + + void add(Runnable task) { + workQueue.add(task); + } + + void stop() { + isStopped = true; + } + + @Override + public void run() { + while (!isDone && !isStopped) { + try { + Runnable task = workQueue.take(); + task.run(); + if (task instanceof TerminalTask) { + isDone = true; + } + } catch (InterruptedException e) { + isStopped = true; + } + } + } + } + + private class RequestCreator implements Runnable { + private final Logger logger = + LoggerFactory.getLogger(RequestCreator.class); + + @Override + public void run() { + logger.trace("Creating request [bucket: {}] [key: {}]", + bucket, key); + CreateMultipartUploadRequest createMultipartUploadRequest = + CreateMultipartUploadRequest.builder() + .bucket(bucket) + .key(key) + .contentType(contentType) + .contentEncoding("UTF-8") + .build(); + CreateMultipartUploadResponse response = + client.createMultipartUpload(createMultipartUploadRequest); + uploadID = response.uploadId(); + } + } + + private class PartUploader implements Runnable { + private final Logger logger = + LoggerFactory.getLogger(PartUploader.class); + + private final ByteArrayOutputStream part; + + PartUploader(ByteArrayOutputStream part) { + this.part = part; + } + + @Override + public void run() { + try { + final int partNumber = partIndex + 1; + + UploadPartRequest uploadPartRequest = UploadPartRequest.builder() + .bucket(bucket) + .key(key) + .uploadId(uploadID) + .partNumber(partNumber) + .build(); + + // There is a small chance that the last part will be empty. + if (part.size() == 0) { + logger.trace("Skipping empty part {} [upload ID: {}]", + partNumber, uploadID); + return; + } + + byte[] bytes = part.toByteArray(); + + logger.trace("Uploading part {} ({} bytes) [upload ID: {}]", + uploadPartRequest.partNumber(), bytes.length, uploadID); + + String etag = client.uploadPart( + uploadPartRequest, + RequestBody.fromBytes(bytes)).eTag(); + CompletedPart completedPart = CompletedPart.builder() + .partNumber(uploadPartRequest.partNumber()) + .eTag(etag) + .build(); + completedParts.add(completedPart); + } finally { + IOUtils.closeQuietly(part); + } + } + } + + private class RequestCompleter implements Runnable, TerminalTask { + private final Logger logger = + LoggerFactory.getLogger(RequestCompleter.class); + + @Override + public void run() { + try { + logger.trace("Completing {}-part request [upload ID: {}]", + completedParts.size(), uploadID); + + CompletedMultipartUpload completedMultipartUpload = + CompletedMultipartUpload.builder() + .parts(completedParts) + .build(); + CompleteMultipartUploadRequest completeMultipartUploadRequest = + CompleteMultipartUploadRequest.builder() + .bucket(bucket) + .key(key) + .uploadId(uploadID) + .multipartUpload(completedMultipartUpload) + .build(); + client.completeMultipartUpload(completeMultipartUploadRequest); + setComplete(true); // CompletableOutputStream method + } catch (S3Exception e) { + logger.warn(e.getMessage()); + } finally { + if (observer != null) { + synchronized (instance) { + instance.notifyAll(); + } + } + } + } + } + + private class RequestAborter implements Runnable, TerminalTask { + private final Logger logger = + LoggerFactory.getLogger(RequestAborter.class); + + @Override + public void run() { + try { + logger.trace("Aborting multipart request [upload ID: {}]", + uploadID); + + AbortMultipartUploadRequest abortMultipartUploadRequest = + AbortMultipartUploadRequest.builder() + .bucket(bucket) + .key(key) + .uploadId(uploadID) + .build(); + client.abortMultipartUpload(abortMultipartUploadRequest); + setComplete(false); + } catch (S3Exception e) { + logger.warn(e.getMessage()); + } finally { + if (observer != null) { + synchronized (instance) { + instance.notifyAll(); + } + } + } + } + } + + /** 5 MB is the minimum allowed by S3 for all but the last part. */ + public static final int MINIMUM_PART_LENGTH = 1024 * 1024 * 5; + + private final S3Client client; + private final String bucket, key, contentType; + + private ByteArrayOutputStream currentPart; + private final List completedParts = new ArrayList<>(); + private final Worker worker = new Worker(); + private boolean requestCreated; + + private String uploadID; + private int partIndex; + private long indexWithinPart; + + /** For an instance to wait for an upload notification during testing. */ + Object observer; + + /** Helps notify {@link #observer} of a completed upload during testing. */ + private final S3MultipartAsyncOutputStream instance; + + /** + * @param client Client. + * @param bucket Target bucket. + * @param key Target key. + * @param contentType Content type of the created object. + */ + public S3MultipartAsyncOutputStream(S3Client client, + String bucket, + String key, + String contentType) { + this.client = client; + this.bucket = bucket; + this.key = key; + this.contentType = contentType; + this.instance = this; + ThreadPool.getInstance().submit(worker); + } + + @Override + public void close() throws IOException { + if (isComplete()) { + worker.add(new PartUploader(getCurrentPart())); + // The worker will exit after running this. + worker.add(new RequestCompleter()); + } else { + // The worker will exit after running this. + worker.add(new RequestAborter()); + } + } + + @Override + public void write(int b) throws IOException { + ByteArrayOutputStream part = getCurrentPart(); + part.write(b); + indexWithinPart++; + createRequestIfNecessary(); + uploadPartIfNecessary(); + } + + @Override + public void write(byte[] b) throws IOException { + ByteArrayOutputStream part = getCurrentPart(); + part.write(b); + indexWithinPart += b.length; + createRequestIfNecessary(); + uploadPartIfNecessary(); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + ByteArrayOutputStream part = getCurrentPart(); + part.write(b, off, len); + indexWithinPart += len; + createRequestIfNecessary(); + uploadPartIfNecessary(); + } + + private ByteArrayOutputStream getCurrentPart() { + if (currentPart == null) { + currentPart = new ByteArrayOutputStream(); + } + return currentPart; + } + + private void createRequestIfNecessary() { + if (!requestCreated) { + worker.add(new RequestCreator()); + requestCreated = true; + } + } + + private void uploadPartIfNecessary() { + if (indexWithinPart >= MINIMUM_PART_LENGTH) { + worker.add(new PartUploader(currentPart)); + IOUtils.closeQuietly(currentPart); + currentPart = null; + indexWithinPart = 0; + partIndex++; + } + } + +} diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStreamTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStreamTest.java new file mode 100644 index 000000000..721ff18f9 --- /dev/null +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStreamTest.java @@ -0,0 +1,311 @@ +package edu.illinois.library.cantaloupe.cache; + +import edu.illinois.library.cantaloupe.test.BaseTest; +import edu.illinois.library.cantaloupe.test.ConfigurationConstants; +import edu.illinois.library.cantaloupe.test.TestUtil; +import edu.illinois.library.cantaloupe.util.S3ClientBuilder; +import edu.illinois.library.cantaloupe.util.S3Utils; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.SecureRandom; + +import static org.junit.jupiter.api.Assertions.*; + +class S3MultipartAsyncOutputStreamTest extends BaseTest { + + private static S3Client client; + + @BeforeAll + public static void beforeClass() throws Exception { + BaseTest.beforeClass(); + S3Utils.createBucket(client(), getBucket()); + } + + @AfterAll + public static void afterClass() throws Exception { + BaseTest.afterClass(); + if (client != null) { + client.close(); + } + } + + private static synchronized S3Client client() { + if (client == null) { + client = new S3ClientBuilder() + .endpointURI(getEndpoint()) + .region(getRegion()) + .accessKeyID(getAccessKeyId()) + .secretAccessKey(getSecretKey()) + .build(); + } + return client; + } + + private static void delete(String key) { + DeleteObjectRequest request = DeleteObjectRequest.builder() + .bucket(getBucket()) + .key(key) + .build(); + client.deleteObject(request); + } + + private static String getAccessKeyId() { + org.apache.commons.configuration.Configuration testConfig = + TestUtil.getTestConfig(); + return testConfig.getString(ConfigurationConstants.S3_ACCESS_KEY_ID.getKey()); + } + + private static String getBucket() { + org.apache.commons.configuration.Configuration testConfig = + TestUtil.getTestConfig(); + return testConfig.getString(ConfigurationConstants.S3_BUCKET.getKey()); + } + + private static URI getEndpoint() { + org.apache.commons.configuration.Configuration testConfig = + TestUtil.getTestConfig(); + String endpointStr = testConfig.getString(ConfigurationConstants.S3_ENDPOINT.getKey()); + if (endpointStr != null && !endpointStr.isBlank()) { + try { + return new URI(endpointStr); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); + } + } + return null; + } + + private static String getRegion() { + org.apache.commons.configuration.Configuration testConfig = + TestUtil.getTestConfig(); + return testConfig.getString(ConfigurationConstants.S3_REGION.getKey()); + } + + private static String getSecretKey() { + org.apache.commons.configuration.Configuration testConfig = + TestUtil.getTestConfig(); + return testConfig.getString(ConfigurationConstants.S3_SECRET_KEY.getKey()); + } + + private static byte[] readBytes(String key) throws IOException { + GetObjectRequest request = GetObjectRequest.builder() + .bucket(getBucket()) + .key(key) + .build(); + ResponseInputStream is = client().getObject(request); + return is.readAllBytes(); + } + + @Test + void closeMarksInstanceComplete() throws Exception { + final String key = S3MultipartAsyncOutputStreamTest.class.getSimpleName() + + "/closeMarksInstanceComplete"; + S3MultipartAsyncOutputStream instance = new S3MultipartAsyncOutputStream( + client(), getBucket(), key, "image/jpeg"); + instance.observer = this; + + try { + byte[] bytes = new byte[1024 * 1024]; + new SecureRandom().nextBytes(bytes); + instance.write(bytes); + instance.close(); + + //noinspection SynchronizationOnLocalVariableOrMethodParameter + synchronized (instance) { + instance.wait(); + } + assertTrue(instance.isComplete()); + } finally { + delete(key); + } + } + + @Test + void write1WithMultipleParts() throws Exception { + final String key = S3MultipartAsyncOutputStreamTest.class.getSimpleName() + + "/write1WithMultipleParts"; + S3MultipartAsyncOutputStream instance = new S3MultipartAsyncOutputStream( + client(), getBucket(), key, "image/jpeg"); + instance.observer = this; + + byte[] expectedBytes = new byte[ + S3MultipartAsyncOutputStream.MINIMUM_PART_LENGTH * 2 + 1024 * 1024]; + new SecureRandom().nextBytes(expectedBytes); + + try { + for (byte b : expectedBytes) { + instance.write(b); + } + instance.close(); + + //noinspection SynchronizationOnLocalVariableOrMethodParameter + synchronized (instance) { + instance.wait(); + } + + byte[] actualBytes = readBytes(key); + assertArrayEquals(expectedBytes, actualBytes); + } finally { + delete(key); + } + } + + @Test + void write1WithSinglePart() throws Exception { + final String key = S3MultipartAsyncOutputStreamTest.class.getSimpleName() + + "/write1WithSinglePart"; + S3MultipartAsyncOutputStream instance = new S3MultipartAsyncOutputStream( + client(), getBucket(), key, "image/jpeg"); + instance.observer = this; + + try { + byte[] expectedBytes = new byte[1024 * 1024]; // smaller than part size + new SecureRandom().nextBytes(expectedBytes); + for (byte b : expectedBytes) { + instance.write(b); + } + instance.close(); + + //noinspection SynchronizationOnLocalVariableOrMethodParameter + synchronized (instance) { + instance.wait(); + } + + byte[] actualBytes = readBytes(key); + assertArrayEquals(expectedBytes, actualBytes); + } finally { + delete(key); + } + } + + /** + * Tests that an object larger than the part size is written correctly. + */ + @Test + void write2WithMultipleParts() throws Exception { + final String key = S3MultipartAsyncOutputStreamTest.class.getSimpleName() + + "/write2WithMultipleParts"; + S3MultipartAsyncOutputStream instance = new S3MultipartAsyncOutputStream( + client(), getBucket(), key, "image/jpeg"); + instance.observer = this; + + byte[] expectedBytes = new byte[ + S3MultipartAsyncOutputStream.MINIMUM_PART_LENGTH * 2 + 1024 * 1024]; + new SecureRandom().nextBytes(expectedBytes); + + try { + instance.write(expectedBytes); + instance.close(); + + //noinspection SynchronizationOnLocalVariableOrMethodParameter + synchronized (instance) { + instance.wait(); + } + + byte[] actualBytes = readBytes(key); + assertArrayEquals(expectedBytes, actualBytes); + } finally { + delete(key); + } + } + + /** + * Tests that an object smaller than the part size is written correctly. + */ + @Test + void write2WithSinglePart() throws Exception { + final String key = S3MultipartAsyncOutputStreamTest.class.getSimpleName() + + "/write2WithSinglePart"; + S3MultipartAsyncOutputStream instance = new S3MultipartAsyncOutputStream( + client(), getBucket(), key, "image/jpeg"); + instance.observer = this; + + try { + byte[] expectedBytes = new byte[1024 * 1024]; // smaller than part size + new SecureRandom().nextBytes(expectedBytes); + instance.write(expectedBytes); + instance.close(); + + //noinspection SynchronizationOnLocalVariableOrMethodParameter + synchronized (instance) { + instance.wait(); + } + + byte[] actualBytes = readBytes(key); + assertArrayEquals(expectedBytes, actualBytes); + } finally { + delete(key); + } + } + + /** + * Tests that an object larger than the part size is written correctly. + */ + @Test + void write3WithMultipleParts() throws Exception { + final String key = S3MultipartAsyncOutputStreamTest.class.getSimpleName() + + "/write3WithMultipleParts"; + S3MultipartAsyncOutputStream instance = new S3MultipartAsyncOutputStream( + client(), getBucket(), key, "image/jpeg"); + instance.observer = this; + + byte[] expectedBytes = new byte[ + S3MultipartAsyncOutputStream.MINIMUM_PART_LENGTH * 2 + 1024 * 1024]; + new SecureRandom().nextBytes(expectedBytes); + + try { + instance.write(expectedBytes, 0, expectedBytes.length); + instance.close(); + + //noinspection SynchronizationOnLocalVariableOrMethodParameter + synchronized (instance) { + instance.wait(); + } + + byte[] actualBytes = readBytes(key); + assertArrayEquals(expectedBytes, actualBytes); + } finally { + delete(key); + } + } + + /** + * Tests that an object smaller than the part size is written correctly. + */ + @Test + void write3WithSinglePart() throws Exception { + final String key = S3MultipartAsyncOutputStreamTest.class.getSimpleName() + + "/write3WithSinglePart"; + S3MultipartAsyncOutputStream instance = new S3MultipartAsyncOutputStream( + client(), getBucket(), key, "image/jpeg"); + instance.observer = this; + + try { + byte[] expectedBytes = new byte[1024 * 1024]; // smaller than part size + new SecureRandom().nextBytes(expectedBytes); + instance.write(expectedBytes, 0, expectedBytes.length); + instance.close(); + + //noinspection SynchronizationOnLocalVariableOrMethodParameter + synchronized (instance) { + instance.wait(); + } + + byte[] actualBytes = readBytes(key); + assertArrayEquals(expectedBytes, actualBytes); + } finally { + delete(key); + } + } + +} From 3ba2c979de713de962a6f9ab15975cb7d7b5a56b Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Fri, 11 Mar 2022 13:53:29 -0600 Subject: [PATCH 022/106] Add an xmp_elements key to the delegate script context --- CHANGES.md | 2 + delegates.rb.sample | 16 +- .../cantaloupe/delegate/JavaContext.java | 13 +- .../library/cantaloupe/image/Metadata.java | 31 +++- .../cantaloupe/image/xmp/MapReader.java | 148 ++++++++++++++++++ .../cantaloupe/image/xmp/package-info.java | 6 + .../cantaloupe/image/MetadataTest.java | 71 +++++++-- .../cantaloupe/image/xmp/MapReaderTest.java | 43 +++++ src/test/resources/xmp/xmp.xmp | 27 ++++ src/test/resources/xmp/xmp2.xmp | 121 ++++++++++++++ 10 files changed, 460 insertions(+), 18 deletions(-) create mode 100644 src/main/java/edu/illinois/library/cantaloupe/image/xmp/MapReader.java create mode 100644 src/main/java/edu/illinois/library/cantaloupe/image/xmp/package-info.java create mode 100644 src/test/java/edu/illinois/library/cantaloupe/image/xmp/MapReaderTest.java create mode 100644 src/test/resources/xmp/xmp.xmp create mode 100644 src/test/resources/xmp/xmp2.xmp diff --git a/CHANGES.md b/CHANGES.md index 88b0d66bc..151cdca4c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,6 +16,8 @@ `delegate_script.pathname` configuration key. * S3Cache uses multipart uploads, which reduces memory usage when caching derivatives larger than 5 MB. +* The delegate script's `metadata` context key contains a new field, + `xmp_elements`, that provides a high-level key-value view of the XMP data. ## 5.0.6 diff --git a/delegates.rb.sample b/delegates.rb.sample index 78f720721..ad96bc633 100644 --- a/delegates.rb.sample +++ b/delegates.rb.sample @@ -366,7 +366,14 @@ class CustomDelegate # "Field2Name": value # ], # "xmp_string": "...", - # "xmp_model": https://jena.apache.org/documentation/javadoc/jena/org/apache/jena/rdf/model/Model.html + # "xmp_model": See https://jena.apache.org/documentation/javadoc/jena/org/apache/jena/rdf/model/Model.html, + # "xmp_elements": { + # "Field1Name": "value", + # "Field2Name": [ + # "value1", + # "value2" + # ] + # }, # "native": { # # structure varies # } @@ -376,10 +383,13 @@ class CustomDelegate # * The `exif` key refers to embedded EXIF data. This also includes IFD0 # metadata from source TIFFs, whether or not an EXIF IFD is present. # * The `iptc` key refers to embedded IPTC IIM data. - # * The `xmp_string` key refers to raw embedded XMP data, which may or may - # not contain EXIF and/or IPTC information. + # * The `xmp_string` key refers to raw embedded XMP data. # * The `xmp_model` key contains a Jena Model object pre-loaded with the # contents of `xmp_string`. + # * The `xmp_elements` key contains a view of the embedded XMP data as key- + # value pairs. This is convenient to use, but may not work correctly with + # all XMP fields--in particular, those that cannot be expressed as + # key-value pairs. # * The `native` key refers to format-specific metadata. # # Any combination of the above keys may be present or missing depending on diff --git a/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaContext.java b/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaContext.java index ad0dd04b3..f460cc415 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaContext.java +++ b/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaContext.java @@ -67,7 +67,14 @@ public interface JavaContext { * "Field2Name": value * ], * "xmp_string": "...", - * "xmp_model": https://jena.apache.org/documentation/javadoc/jena/org/apache/jena/rdf/model/Model.html + * "xmp_model": See https://jena.apache.org/documentation/javadoc/jena/org/apache/jena/rdf/model/Model.html, + * "xmp_elements": { + * "Field1Name": "value", + * "Field2Name": [ + * "value1", + * "value2" + * ] + * }, * "native": { * # structure varies * } @@ -83,6 +90,10 @@ public interface JavaContext { *
    11. The {@code xmp_model} key contains a {@link * org.apache.jena.rdf.model.Model} object pre-loaded with the * contents of {@code xmp_string}.
      12. + *
    12. The {@code xmp_elements} key contains a view of the embedded XMP + * data as key-value pairs. This is convenient to use, but may not work + * correctly with all XMP fields—in particular, those that cannot + * be expressed as key-value pairs. *
    13. The {@code native} key refers to format-specific metadata.
      14. * * diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java b/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java index c4e2d413a..adc208e5b 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java +++ b/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java @@ -8,6 +8,7 @@ import edu.illinois.library.cantaloupe.image.exif.Directory; import edu.illinois.library.cantaloupe.image.exif.Tag; import edu.illinois.library.cantaloupe.image.iptc.DataSet; +import edu.illinois.library.cantaloupe.image.xmp.MapReader; import edu.illinois.library.cantaloupe.util.StringUtils; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; @@ -17,6 +18,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.io.StringReader; import java.nio.charset.StandardCharsets; import java.util.Arrays; @@ -170,16 +172,32 @@ private void readOrientationFromXMP() { } /** - * Returns an RDF/XML string in UTF-8 encoding. The root element is - * {@literal rdf:RDF}, and there is no packet wrapper. - * - * @return XMP data packet. + * @return RDF/XML string in UTF-8 encoding. The root element is {@literal + * rdf:RDF}, and there is no packet wrapper. */ @JsonProperty public Optional getXMP() { return Optional.ofNullable(xmp); } + /** + * @return Map of elements found in the XMP data. If none are found, the + * map is empty. + */ + @JsonIgnore + public Map getXMPElements() { + loadXMP(); + if (xmpModel != null) { + try { + MapReader reader = new MapReader(xmpModel); + return reader.readElements(); + } catch (IOException e) { + LOGGER.warn("getXMPElements(): {}", e.getMessage()); + } + } + return Collections.emptyMap(); + } + /** * @return XMP model backed by the contents of {@link #getXMP()}. */ @@ -290,7 +308,9 @@ public void setXMP(String xmp) { * { * "exif": See {@link Directory#toMap()}, * "iptc": See {@link DataSet#toMap()}, - * "xmp": "...", + * "xmp_string": "...", + * "xmp_model": [Jena model], + * "xmp_elements": {@link Map} * "native": String * }} * @@ -314,6 +334,7 @@ public Map toMap() { // XMP getXMP().ifPresent(xmp -> map.put("xmp_string", xmp)); getXMPModel().ifPresent(model -> map.put("xmp_model", model)); + map.put("xmp_elements", getXMPElements()); // Native metadata getNativeMetadata().ifPresent(nm -> map.put("native", nm)); return Collections.unmodifiableMap(map); diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/xmp/MapReader.java b/src/main/java/edu/illinois/library/cantaloupe/image/xmp/MapReader.java new file mode 100644 index 000000000..5a021ac7b --- /dev/null +++ b/src/main/java/edu/illinois/library/cantaloupe/image/xmp/MapReader.java @@ -0,0 +1,148 @@ +package edu.illinois.library.cantaloupe.image.xmp; + +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.rdf.model.RDFNode; +import org.apache.jena.rdf.model.Statement; +import org.apache.jena.rdf.model.StmtIterator; +import org.apache.jena.riot.RIOT; +import org.apache.jena.riot.RiotException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * Extracts label-value pairs from an XMP model, making them available in a + * {@link Map}. + * + * @author Alex Dolski UIUC + * @since 6.0 + */ +public final class MapReader { + + private static final Logger LOGGER = + LoggerFactory.getLogger(MapReader.class); + + private static final Map PREFIXES = Map.ofEntries( + Map.entry("http://ns.adobe.com/camera-raw-settings/1.0/", "crs"), + Map.entry("http://purl.org/dc/elements/1.1/", "dc"), + Map.entry("http://purl.org/dc/terms/", "dcterms"), + Map.entry("http://ns.adobe.com/exif/1.0/", "exif"), + Map.entry("http://iptc.org/std/Iptc4xmpCore/1.0/xmlns/", "Iptc4xmpCore"), + Map.entry("http://ns.adobe.com/iX/1.0/", "iX"), + Map.entry("http://ns.adobe.com/pdf/1.3/", "pdf"), + Map.entry("http://ns.adobe.com/photoshop/1.0/", "photoshop"), + Map.entry("http://ns.adobe.com/tiff/1.0/", "tiff"), + Map.entry("http://ns.adobe.com/xap/1.0/", "xmp"), + Map.entry("http://ns.adobe.com/xap/1.0/bj/", "xmpBJ"), + Map.entry("http://ns.adobe.com/xmp/1.0/DynamicMedia/", "xmpDM"), + Map.entry("http://ns.adobe.com/xmp/identifier/qual/1.0/", "xmpidq"), + Map.entry("http://ns.adobe.com/xap/1.0/mm/", "xmpMM"), + Map.entry("http://ns.adobe.com/xap/1.0/rights/", "xmpRights"), + Map.entry("http://ns.adobe.com/xap/1.0/t/pg/", "xmpTPg")); + + private final Model model; + private final Map elements = new TreeMap<>(); + private boolean hasReadElements; + + /** + * @param xmp XMP string. {@code } must be the root element. + * @see edu.illinois.library.cantaloupe.util.StringUtils#trimXMP + */ + public MapReader(String xmp) throws IOException { + RIOT.init(); + this.model = ModelFactory.createDefaultModel(); + try (StringReader reader = new StringReader(xmp)) { + model.read(reader, null, "RDF/XML"); + } catch (RiotException | NullPointerException e) { + // The XMP string may be invalid RDF/XML, or there may be a bug + // in Jena (that would be the NPE). Not much we can do. + throw new IOException(e); + } + } + + /** + * @param model XMP model, already initialized. + */ + public MapReader(Model model) { + this.model = model; + } + + public Map readElements() throws IOException { + if (!hasReadElements) { + StmtIterator it = model.listStatements(); + while (it.hasNext()) { + Statement stmt = it.next(); + //System.out.println(stmt.getSubject() + " " + stmt.getSubject().isAnon()); + //System.out.println(" " + stmt.getPredicate()); + //System.out.println(" " + stmt.getObject() + " " + stmt.getObject().isLiteral()); + //System.out.println("---------------------------"); + if (!stmt.getSubject().isAnon()) { + recurse(stmt); + } + } + LOGGER.trace("readElements(): read {} elements", elements.size()); + hasReadElements = true; + } + return Collections.unmodifiableMap(elements); + } + + private void recurse(Statement stmt) { + recurse(stmt, null); + } + + private void recurse(Statement stmt, String predicateOverride) { + String predicate = stmt.getPredicate().toString(); + if (stmt.getObject().isLiteral()) { + addElement(label(predicateOverride != null ? predicateOverride : predicate), + stmt.getObject().asLiteral().getValue()); + } else { + StmtIterator it = model.listStatements( + stmt.getObject().asResource(), null, (RDFNode) null); + while (it.hasNext()) { + Statement substmt = it.next(); + predicateOverride = null; + if (substmt.getPredicate().toString().matches("(.*)#_\\d+\\b")) { + predicateOverride = predicate; + } + recurse(substmt, predicateOverride); + } + } + } + + private void addElement(String label, Object value) { + if (elements.containsKey(label)) { + if (elements.get(label) instanceof List) { + @SuppressWarnings("unchecked") + List valueList = (List) elements.get(label); + valueList.add(value); + } else { + List valueList = new ArrayList<>(); + valueList.add(elements.get(label)); + valueList.add(value); + elements.put(label, valueList); + } + } else { + elements.put(label, value); + } + } + + private String label(String uri) { + for (Map.Entry entry : PREFIXES.entrySet()) { + if (uri.startsWith(entry.getKey())) { + String prefix = entry.getValue(); + String[] parts = uri.split("/"); + return prefix + ":" + parts[parts.length - 1]; + } + } + return uri; + } + +} \ No newline at end of file diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/xmp/package-info.java b/src/main/java/edu/illinois/library/cantaloupe/image/xmp/package-info.java new file mode 100644 index 000000000..acd0e10b0 --- /dev/null +++ b/src/main/java/edu/illinois/library/cantaloupe/image/xmp/package-info.java @@ -0,0 +1,6 @@ +/** + *

      Contains an XMP parser.

      + * + * @since 6.0 + */ +package edu.illinois.library.cantaloupe.image.xmp; \ No newline at end of file diff --git a/src/test/java/edu/illinois/library/cantaloupe/image/MetadataTest.java b/src/test/java/edu/illinois/library/cantaloupe/image/MetadataTest.java index 654ed56f7..4f993490a 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/image/MetadataTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/image/MetadataTest.java @@ -15,6 +15,7 @@ import org.junit.jupiter.api.Test; import java.nio.file.Path; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -32,17 +33,21 @@ public void setUp() throws Exception { instance = new Metadata(); } + /* encapsulateXMP() */ + @Test void testEncapsulateXMP() { final String xmp = ""; - String actual = Metadata.encapsulateXMP(xmp); + String actual = Metadata.encapsulateXMP(xmp); assertTrue(actual.startsWith("")); } + /* equals() */ + @Test void testEqualsWithEqualInstances() { - Directory exif = new Directory(TagSet.EXIF); + Directory exif = new Directory(TagSet.EXIF); List iptc = List.of(new DataSet( edu.illinois.library.cantaloupe.image.iptc.Tag.CITY, "Urbana".getBytes())); @@ -117,9 +122,11 @@ void testEqualsWithDifferentXMP() { assertNotEquals(m1, m2); } + /* getEXIF() */ + @Test void testGetEXIFWithPresentEXIFData() throws Exception { - Path fixture = TestUtil.getImage("jpg-exif.jpg"); + Path fixture = TestUtil.getImage("jpg-exif.jpg"); ImageReader reader = new ImageReaderFactory() .newImageReader(Format.get("jpg"), fixture); try { @@ -132,7 +139,7 @@ void testGetEXIFWithPresentEXIFData() throws Exception { @Test void testGetEXIFWithNoEXIFData() throws Exception { - Path fixture = TestUtil.getImage("jpg"); + Path fixture = TestUtil.getImage("jpg"); ImageReader reader = new ImageReaderFactory() .newImageReader(Format.get("jpg"), fixture); try { @@ -143,9 +150,11 @@ void testGetEXIFWithNoEXIFData() throws Exception { } } + /* getIPTC() */ + @Test void testGetIPTCWithPresentIPTCData() throws Exception { - Path fixture = TestUtil.getImage("jpg-iptc.jpg"); + Path fixture = TestUtil.getImage("jpg-iptc.jpg"); ImageReader reader = new ImageReaderFactory() .newImageReader(Format.get("jpg"), fixture); try { @@ -158,7 +167,7 @@ void testGetIPTCWithPresentIPTCData() throws Exception { @Test void testGetIPTCWithNoIPTCData() throws Exception { - Path fixture = TestUtil.getImage("jpg"); + Path fixture = TestUtil.getImage("jpg"); ImageReader reader = new ImageReaderFactory() .newImageReader(Format.get("jpg"), fixture); try { @@ -169,6 +178,8 @@ void testGetIPTCWithNoIPTCData() throws Exception { } } + /* getNativeMetadata() */ + @Test void testGetNativeMetadataWithPresentData() throws Exception { Path fixture = TestUtil.getImage("png-nativemetadata.png"); @@ -247,9 +258,40 @@ void testGetOrientationWithMalformedXMP() { assertEquals(Orientation.ROTATE_0, instance.getOrientation()); } + /* getXMPElements() */ + + @Test + void testGetXMPElementsWithPresentXMPData() throws Exception { + Path fixture = TestUtil.getImage("jpg-xmp.jpg"); + ImageReader reader = new ImageReaderFactory() + .newImageReader(Format.get("jpg"), fixture); + try { + Metadata metadata = reader.getMetadata(0); + Map model = metadata.getXMPElements(); + assertEquals(6, model.size()); + } finally { + reader.dispose(); + } + } + + @Test + void testGetXMPElementsWithNoXMPData() throws Exception { + Path fixture = TestUtil.getImage("jpg"); + ImageReader reader = new ImageReaderFactory() + .newImageReader(Format.get("jpg"), fixture); + try { + Metadata metadata = reader.getMetadata(0); + assertTrue(metadata.getXMPElements().isEmpty()); + } finally { + reader.dispose(); + } + } + + /* getXMPModel() */ + @Test void testGetXMPModelWithPresentXMPData() throws Exception { - Path fixture = TestUtil.getImage("jpg-xmp.jpg"); + Path fixture = TestUtil.getImage("jpg-xmp.jpg"); ImageReader reader = new ImageReaderFactory() .newImageReader(Format.get("jpg"), fixture); try { @@ -263,7 +305,7 @@ void testGetXMPModelWithPresentXMPData() throws Exception { @Test void testGetXMPModelWithNoXMPData() throws Exception { - Path fixture = TestUtil.getImage("jpg"); + Path fixture = TestUtil.getImage("jpg"); ImageReader reader = new ImageReaderFactory() .newImageReader(Format.get("jpg"), fixture); try { @@ -274,10 +316,12 @@ void testGetXMPModelWithNoXMPData() throws Exception { } } + /* hashCode() */ + @Test void testHashCodeWithEqualInstances() { Directory exif = new Directory(TagSet.EXIF); - String xmp = "cats"; + String xmp = "cats"; Metadata m1 = new Metadata(); m1.setEXIF(exif); @@ -346,18 +390,24 @@ void testHashCodeWithDifferentXMP() { assertNotEquals(m1.hashCode(), m2.hashCode()); } + /* setEXIF() */ + @Test void testSetEXIFWithNullArgument() { instance.setEXIF(null); assertFalse(instance.getEXIF().isPresent()); } + /* setIPTC() */ + @Test void testSetIPTCWithNullArgument() { instance.setIPTC(null); assertFalse(instance.getIPTC().isPresent()); } + /* setXMP() */ + @Test void testSetXMPWithNullByteArrayArgument() { instance.setXMP((byte[]) null); @@ -378,6 +428,8 @@ void testSetXMPTrimsData() { assertTrue(xmp.endsWith("")); } + /* toMap() */ + @Test void testToMap() { // assemble the expected map structure @@ -402,6 +454,7 @@ void testToMap() { edu.illinois.library.cantaloupe.image.iptc.Tag.CITY, "Urbana".getBytes()).toMap())); expectedMap.put("xmp_string", ""); + expectedMap.put("xmp_elements", Collections.emptyMap()); expectedMap.put("native", Map.of("key1", "value1", "key2", "value2")); // assemble the Metadata diff --git a/src/test/java/edu/illinois/library/cantaloupe/image/xmp/MapReaderTest.java b/src/test/java/edu/illinois/library/cantaloupe/image/xmp/MapReaderTest.java new file mode 100644 index 000000000..a9d6c33d0 --- /dev/null +++ b/src/test/java/edu/illinois/library/cantaloupe/image/xmp/MapReaderTest.java @@ -0,0 +1,43 @@ +package edu.illinois.library.cantaloupe.image.xmp; + +import edu.illinois.library.cantaloupe.test.TestUtil; +import edu.illinois.library.cantaloupe.util.StringUtils; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class MapReaderTest { + + @Test + void readElements1() throws Exception { + String xmp = Files.readString(TestUtil.getFixture("xmp/xmp.xmp")); + xmp = StringUtils.trimXMP(xmp); + MapReader reader = new MapReader(xmp); + Map elements = reader.readElements(); + + print(elements); + + assertEquals(18, elements.size()); + } + + @Test + void readElements2() throws Exception { + String xmp = Files.readString(TestUtil.getFixture("xmp/xmp2.xmp")); + xmp = StringUtils.trimXMP(xmp); + MapReader reader = new MapReader(xmp); + Map elements = reader.readElements(); + assertEquals(61, elements.size()); + } + + private static void print(Map elements) { + System.out.println("------ ELEMENTS -------"); + for (Map.Entry entry : elements.entrySet()) { + System.out.println(entry.getKey()); + System.out.println(entry.getValue()); + System.out.println("-------------"); + } + } +} \ No newline at end of file diff --git a/src/test/resources/xmp/xmp.xmp b/src/test/resources/xmp/xmp.xmp new file mode 100644 index 000000000..4364c491c --- /dev/null +++ b/src/test/resources/xmp/xmp.xmp @@ -0,0 +1,27 @@ + + + + + + Illini Union Photographs Record Series 3707005 + + + + + University of Illinois Library + + + + + + Preservation Master creation + new subject code + + + + + No reproduction without prior permission + + + + \ No newline at end of file diff --git a/src/test/resources/xmp/xmp2.xmp b/src/test/resources/xmp/xmp2.xmp new file mode 100644 index 000000000..979ee87d6 --- /dev/null +++ b/src/test/resources/xmp/xmp2.xmp @@ -0,0 +1,121 @@ + + + + + 3.0 + CRW_1101.CRW + Custom + 5800 + +11 + +1.15 + 0 + 39 + +60 + 0 + 25 + 0 + 15 + 0 + 0 + 0 + 0 + -1 + +24 + -12 + -18 + +5 + -13 + Custom + + + 0, 0 + 34, 16 + 70, 53 + 121, 122 + 184, 214 + 255, 255 + + + ACR 2.4 + True + False + + + 1/100 + 6643856/1000000 + 8/1 + 6/1 + 2 + 2004-08-12T06:57:52-07:00 + 0/1 + 4/1 + 5 + 28/1 + + + 200 + + + + False + + + + 0660217086 + 18.0-55.0 mm + + + + + Canon + Canon EOS DIGITAL REBEL + 6 + 3072 + 2048 + 2 + 240/1 + 240/1 + 2 + + + 16 + 16 + 16 + + + 2004-08-12T06:57:52-07:00 + + + 2004-08-12T06:57:52-07:00 + 2005-03-25T16:27:25-08:00 + 0 + + + True + + + + + Bruce Fraser + + + + + ©2004 Bruce Fraser. All Rights Reserved + + + + + Glencoe + Scotland + Flowers + + + + + lichen and heather in Glencoe + + + + + \ No newline at end of file From 61dfdf51b1d514aa866bf9c98d52c56927d53e96 Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Fri, 11 Mar 2022 13:57:06 -0600 Subject: [PATCH 023/106] Move e.i.l.c.image.Metadata.encapsulateXMP() to e.i.l.c.image.xmp.Utils --- .../library/cantaloupe/image/Metadata.java | 24 -------------- .../library/cantaloupe/image/xmp/Utils.java | 31 +++++++++++++++++++ .../processor/codec/gif/GIFImageWriter.java | 3 +- .../cantaloupe/processor/codec/jpeg/Util.java | 4 +-- .../cantaloupe/image/MetadataTest.java | 11 +------ .../cantaloupe/image/xmp/UtilsTest.java | 20 ++++++++++++ .../codec/jpeg/TurboJPEGImageWriterTest.java | 4 +-- 7 files changed, 58 insertions(+), 39 deletions(-) create mode 100644 src/main/java/edu/illinois/library/cantaloupe/image/xmp/Utils.java create mode 100644 src/test/java/edu/illinois/library/cantaloupe/image/xmp/UtilsTest.java diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java b/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java index adc208e5b..94ae2360f 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java +++ b/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java @@ -47,9 +47,6 @@ public class Metadata { private static final String XMP_ORIENTATION_PREDICATE = "http://ns.adobe.com/tiff/1.0/Orientation"; - private static final String XMP_TOOLKIT = Application.getName() + " " + - Application.getVersion(); - protected Directory exif; protected List iptcDataSets; protected String xmp; @@ -65,27 +62,6 @@ public class Metadata { */ private transient Orientation orientation; - /** - * Returns an XMP string encapsulated in an {@literal x:xmpmeta} element, - * which is itself encapsulated in an {@literal xpacket} PI. - * - * @param xmp XMP string with an {@literal rdf:RDF} root element. - * @return Encapsulated XMP data packet. - */ - public static String encapsulateXMP(String xmp) { - final StringBuilder b = new StringBuilder(); - b.append(""); - b.append(""); - b.append(xmp); - b.append(""); - // Append the magic trailer - b.append(" ".repeat(2048)); - b.append(""); - return b.toString(); - } - @Override public boolean equals(Object obj) { if (obj == this) { diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/xmp/Utils.java b/src/main/java/edu/illinois/library/cantaloupe/image/xmp/Utils.java new file mode 100644 index 000000000..edaef8fe8 --- /dev/null +++ b/src/main/java/edu/illinois/library/cantaloupe/image/xmp/Utils.java @@ -0,0 +1,31 @@ +package edu.illinois.library.cantaloupe.image.xmp; + +import edu.illinois.library.cantaloupe.Application; + +public final class Utils { + + private static final String XMP_TOOLKIT = Application.getName() + " " + + Application.getVersion(); + + /** + * Returns an XMP string encapsulated in an {@literal x:xmpmeta} element, + * which is itself encapsulated in an {@literal xpacket} PI. + * + * @param xmp XMP string with an {@literal rdf:RDF} root element. + * @return Encapsulated XMP data packet. + */ + public static String encapsulateXMP(String xmp) { + final StringBuilder b = new StringBuilder(); + b.append(""); + b.append(""); + b.append(xmp); + b.append(""); + // Append the magic trailer + b.append(" ".repeat(2048)); + b.append(""); + return b.toString(); + } + +} diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/gif/GIFImageWriter.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/gif/GIFImageWriter.java index 0a4627f40..347d57e78 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/gif/GIFImageWriter.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/gif/GIFImageWriter.java @@ -2,6 +2,7 @@ import edu.illinois.library.cantaloupe.config.Configuration; import edu.illinois.library.cantaloupe.image.Metadata; +import edu.illinois.library.cantaloupe.image.xmp.Utils; import edu.illinois.library.cantaloupe.processor.codec.AbstractIIOImageWriter; import edu.illinois.library.cantaloupe.processor.codec.BufferedImageSequence; import edu.illinois.library.cantaloupe.processor.codec.ImageWriter; @@ -62,7 +63,7 @@ protected void addMetadata(final IIOMetadataNode baseTree) { final Metadata metadata = encode.getMetadata(); if (metadata != null) { metadata.getXMP().ifPresent(xmp -> { - xmp = Metadata.encapsulateXMP(xmp); + xmp = Utils.encapsulateXMP(xmp); // Get the /ApplicationExtensions node, creating it if necessary. final NodeList appExtensionsList = diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/Util.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/Util.java index 5a09e5bc0..eaf45762a 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/Util.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/Util.java @@ -1,6 +1,6 @@ package edu.illinois.library.cantaloupe.processor.codec.jpeg; -import edu.illinois.library.cantaloupe.image.Metadata; +import edu.illinois.library.cantaloupe.image.xmp.Utils; import edu.illinois.library.cantaloupe.util.ArrayUtils; import edu.illinois.library.cantaloupe.util.StringUtils; import org.apache.jena.rdf.model.Model; @@ -28,7 +28,7 @@ static byte[] assembleAPP1Segment(String xmp) { try { final ByteArrayOutputStream os = new ByteArrayOutputStream(); final byte[] headerBytes = Constants.STANDARD_XMP_SEGMENT_HEADER; - final byte[] xmpBytes = Metadata.encapsulateXMP(xmp). + final byte[] xmpBytes = Utils.encapsulateXMP(xmp). getBytes(StandardCharsets.UTF_8); // write segment marker os.write(Marker.APP1.marker()); diff --git a/src/test/java/edu/illinois/library/cantaloupe/image/MetadataTest.java b/src/test/java/edu/illinois/library/cantaloupe/image/MetadataTest.java index 4f993490a..366c9cb77 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/image/MetadataTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/image/MetadataTest.java @@ -6,6 +6,7 @@ import edu.illinois.library.cantaloupe.image.exif.Tag; import edu.illinois.library.cantaloupe.image.exif.TagSet; import edu.illinois.library.cantaloupe.image.iptc.DataSet; +import edu.illinois.library.cantaloupe.image.xmp.Utils; import edu.illinois.library.cantaloupe.processor.codec.ImageReader; import edu.illinois.library.cantaloupe.processor.codec.ImageReaderFactory; import edu.illinois.library.cantaloupe.test.BaseTest; @@ -33,16 +34,6 @@ public void setUp() throws Exception { instance = new Metadata(); } - /* encapsulateXMP() */ - - @Test - void testEncapsulateXMP() { - final String xmp = ""; - String actual = Metadata.encapsulateXMP(xmp); - assertTrue(actual.startsWith("")); - } - /* equals() */ @Test diff --git a/src/test/java/edu/illinois/library/cantaloupe/image/xmp/UtilsTest.java b/src/test/java/edu/illinois/library/cantaloupe/image/xmp/UtilsTest.java new file mode 100644 index 000000000..62dbd11c2 --- /dev/null +++ b/src/test/java/edu/illinois/library/cantaloupe/image/xmp/UtilsTest.java @@ -0,0 +1,20 @@ +package edu.illinois.library.cantaloupe.image.xmp; + +import edu.illinois.library.cantaloupe.test.BaseTest; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class UtilsTest extends BaseTest { + + /* encapsulateXMP() */ + + @Test + void testEncapsulateXMP() { + final String xmp = ""; + String actual = Utils.encapsulateXMP(xmp); + assertTrue(actual.startsWith("")); + } + +} \ No newline at end of file diff --git a/src/test/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriterTest.java b/src/test/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriterTest.java index dd998a04d..74a752a69 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriterTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriterTest.java @@ -1,7 +1,7 @@ package edu.illinois.library.cantaloupe.processor.codec.jpeg; -import edu.illinois.library.cantaloupe.image.Metadata; import edu.illinois.library.cantaloupe.image.Rectangle; +import edu.illinois.library.cantaloupe.image.xmp.Utils; import edu.illinois.library.cantaloupe.test.BaseTest; import edu.illinois.library.cantaloupe.test.TestUtil; import edu.illinois.library.cantaloupe.util.Rational; @@ -104,7 +104,7 @@ public void testSetXMP() throws Exception { final ByteArrayOutputStream expectedSegment = new ByteArrayOutputStream(); final byte[] headerBytes = "http://ns.adobe.com/xap/1.0/\0".getBytes(); - final byte[] xmpBytes = Metadata.encapsulateXMP(xmp). + final byte[] xmpBytes = Utils.encapsulateXMP(xmp). getBytes(StandardCharsets.UTF_8); // write segment marker expectedSegment.write(new byte[]{(byte) 0xff, (byte) 0xe1}); From 23ac283e3a479bed52d05fa443a69f537bae4024 Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Fri, 11 Mar 2022 14:00:03 -0600 Subject: [PATCH 024/106] Move e.i.l.c.util.StringUtils.trimXMP() to e.i.l.c.image.xmp.Utils --- .../library/cantaloupe/image/Metadata.java | 5 ++-- .../cantaloupe/image/xmp/MapReader.java | 2 +- .../library/cantaloupe/image/xmp/Utils.java | 13 +++++++++ .../processor/codec/gif/GIFMetadata.java | 4 +-- .../cantaloupe/processor/codec/jpeg/Util.java | 5 ++-- .../jpeg2000/JPEG2000KakaduImageReader.java | 4 +-- .../processor/codec/png/PNGMetadata.java | 4 +-- .../processor/codec/tiff/TIFFMetadata.java | 4 +-- .../library/cantaloupe/util/StringUtils.java | 13 --------- .../cantaloupe/image/xmp/MapReaderTest.java | 5 ++-- .../cantaloupe/image/xmp/UtilsTest.java | 28 +++++++++++++++++++ .../cantaloupe/util/StringUtilsTest.java | 26 ----------------- 12 files changed, 56 insertions(+), 57 deletions(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java b/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java index 94ae2360f..1baa942c2 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java +++ b/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java @@ -4,12 +4,11 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import edu.illinois.library.cantaloupe.Application; import edu.illinois.library.cantaloupe.image.exif.Directory; import edu.illinois.library.cantaloupe.image.exif.Tag; import edu.illinois.library.cantaloupe.image.iptc.DataSet; import edu.illinois.library.cantaloupe.image.xmp.MapReader; -import edu.illinois.library.cantaloupe.util.StringUtils; +import edu.illinois.library.cantaloupe.image.xmp.Utils; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.rdf.model.NodeIterator; @@ -269,7 +268,7 @@ public void setXMP(byte[] xmp) { */ public void setXMP(String xmp) { if (xmp != null) { - this.xmp = StringUtils.trimXMP(xmp); + this.xmp = Utils.trimXMP(xmp); } else { this.xmp = null; this.xmpModel = null; diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/xmp/MapReader.java b/src/main/java/edu/illinois/library/cantaloupe/image/xmp/MapReader.java index 5a021ac7b..51b608fe6 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/image/xmp/MapReader.java +++ b/src/main/java/edu/illinois/library/cantaloupe/image/xmp/MapReader.java @@ -54,7 +54,7 @@ public final class MapReader { /** * @param xmp XMP string. {@code } must be the root element. - * @see edu.illinois.library.cantaloupe.util.StringUtils#trimXMP + * @see Utils#trimXMP */ public MapReader(String xmp) throws IOException { RIOT.init(); diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/xmp/Utils.java b/src/main/java/edu/illinois/library/cantaloupe/image/xmp/Utils.java index edaef8fe8..747d63aef 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/image/xmp/Utils.java +++ b/src/main/java/edu/illinois/library/cantaloupe/image/xmp/Utils.java @@ -28,4 +28,17 @@ public static String encapsulateXMP(String xmp) { return b.toString(); } + /** + * Strips any enclosing tags or other content around the {@literal rdf:RDF} + * element within an RDF/XML XMP string. + */ + public static String trimXMP(String xmp) { + final int start = xmp.indexOf(" -1 && end > -1) { + xmp = xmp.substring(start, end + 10); + } + return xmp; + } + } diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/gif/GIFMetadata.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/gif/GIFMetadata.java index 945db16e5..9640511b0 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/gif/GIFMetadata.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/gif/GIFMetadata.java @@ -2,7 +2,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import edu.illinois.library.cantaloupe.image.Metadata; -import edu.illinois.library.cantaloupe.util.StringUtils; +import edu.illinois.library.cantaloupe.image.xmp.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,7 +71,7 @@ public Optional getXMP() { try { xmp = reader.getXMP(); if (xmp != null) { - xmp = StringUtils.trimXMP(xmp); + xmp = Utils.trimXMP(xmp); } } catch (IOException e) { LOGGER.warn("getXMP(): {}", e.getMessage()); diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/Util.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/Util.java index eaf45762a..b1fce36b0 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/Util.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/Util.java @@ -2,7 +2,6 @@ import edu.illinois.library.cantaloupe.image.xmp.Utils; import edu.illinois.library.cantaloupe.util.ArrayUtils; -import edu.illinois.library.cantaloupe.util.StringUtils; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.rdf.model.RDFNode; @@ -60,12 +59,12 @@ static String assembleXMP(final List xmpChunks) throws IOException { final int numChunks = xmpChunks.size(); if (numChunks > 0) { standardXMP = new String(xmpChunks.get(0), StandardCharsets.UTF_8); - standardXMP = StringUtils.trimXMP(standardXMP); + standardXMP = Utils.trimXMP(standardXMP); if (numChunks > 1) { String extendedXMP = new String( mergeChunks(xmpChunks.subList(1, numChunks)), StandardCharsets.UTF_8); - extendedXMP = StringUtils.trimXMP(extendedXMP); + extendedXMP = Utils.trimXMP(extendedXMP); return mergeXMPModels(standardXMP, extendedXMP); } } diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg2000/JPEG2000KakaduImageReader.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg2000/JPEG2000KakaduImageReader.java index 2ab574dcd..4db63c068 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg2000/JPEG2000KakaduImageReader.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg2000/JPEG2000KakaduImageReader.java @@ -2,11 +2,11 @@ import edu.illinois.library.cantaloupe.image.Rectangle; import edu.illinois.library.cantaloupe.image.ScaleConstraint; +import edu.illinois.library.cantaloupe.image.xmp.Utils; import edu.illinois.library.cantaloupe.operation.ReductionFactor; import edu.illinois.library.cantaloupe.operation.Scale; import edu.illinois.library.cantaloupe.processor.SourceFormatException; import edu.illinois.library.cantaloupe.util.Stopwatch; -import edu.illinois.library.cantaloupe.util.StringUtils; import kdu_jni.Jp2_threadsafe_family_src; import kdu_jni.Jpx_codestream_source; import kdu_jni.Jpx_input_box; @@ -527,7 +527,7 @@ private void readMetadata() throws IOException { iptc = Arrays.copyOfRange(buffer, 16, buffer.length); } else if (isXMP) { xmp = new String(buffer, StandardCharsets.UTF_8); - xmp = StringUtils.trimXMP(xmp); + xmp = Utils.trimXMP(xmp); } } finally { box.Close(); diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/png/PNGMetadata.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/png/PNGMetadata.java index 3e2af6ab4..830e5a934 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/png/PNGMetadata.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/png/PNGMetadata.java @@ -1,7 +1,7 @@ package edu.illinois.library.cantaloupe.processor.codec.png; +import edu.illinois.library.cantaloupe.image.xmp.Utils; import edu.illinois.library.cantaloupe.processor.codec.IIOMetadata; -import edu.illinois.library.cantaloupe.util.StringUtils; import org.w3c.dom.NodeList; import javax.imageio.metadata.IIOMetadataNode; @@ -71,7 +71,7 @@ public Optional getXMP() { .getAttribute("text") .getBytes(Charset.forName("UTF-8")); xmp = new String(xmpBytes, StandardCharsets.UTF_8); - xmp = StringUtils.trimXMP(xmp); + xmp = Utils.trimXMP(xmp); } } } diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/tiff/TIFFMetadata.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/tiff/TIFFMetadata.java index db2a8ec9a..077ab2517 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/tiff/TIFFMetadata.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/tiff/TIFFMetadata.java @@ -3,8 +3,8 @@ import edu.illinois.library.cantaloupe.image.exif.Directory; import edu.illinois.library.cantaloupe.image.iptc.DataSet; import edu.illinois.library.cantaloupe.image.iptc.Reader; +import edu.illinois.library.cantaloupe.image.xmp.Utils; import edu.illinois.library.cantaloupe.processor.codec.IIOMetadata; -import edu.illinois.library.cantaloupe.util.StringUtils; import it.geosolutions.imageio.plugins.tiff.TIFFDirectory; import it.geosolutions.imageio.plugins.tiff.TIFFField; import org.slf4j.Logger; @@ -97,7 +97,7 @@ public Optional getXMP() { xmp = new String( (byte[]) xmpField.getData(), StandardCharsets.UTF_8); - xmp = StringUtils.trimXMP(xmp); + xmp = Utils.trimXMP(xmp); } } return Optional.ofNullable(xmp); diff --git a/src/main/java/edu/illinois/library/cantaloupe/util/StringUtils.java b/src/main/java/edu/illinois/library/cantaloupe/util/StringUtils.java index c8e4c38c5..015bf0d60 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/util/StringUtils.java +++ b/src/main/java/edu/illinois/library/cantaloupe/util/StringUtils.java @@ -217,19 +217,6 @@ public static long toByteSize(String str) { return Math.round(number * Math.pow(1024, exponent)); } - /** - * Strips any enclosing tags or other content around the {@literal rdf:RDF} - * element within an RDF/XML XMP string. - */ - public static String trimXMP(String xmp) { - final int start = xmp.indexOf(" -1 && end > -1) { - xmp = xmp.substring(start, end + 10); - } - return xmp; - } - /** * Returns an array of strings, one for each line in the string after it * has been wrapped to fit lines of maxWidth. Lines end with any diff --git a/src/test/java/edu/illinois/library/cantaloupe/image/xmp/MapReaderTest.java b/src/test/java/edu/illinois/library/cantaloupe/image/xmp/MapReaderTest.java index a9d6c33d0..ab440fbc9 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/image/xmp/MapReaderTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/image/xmp/MapReaderTest.java @@ -1,7 +1,6 @@ package edu.illinois.library.cantaloupe.image.xmp; import edu.illinois.library.cantaloupe.test.TestUtil; -import edu.illinois.library.cantaloupe.util.StringUtils; import org.junit.jupiter.api.Test; import java.nio.file.Files; @@ -14,7 +13,7 @@ class MapReaderTest { @Test void readElements1() throws Exception { String xmp = Files.readString(TestUtil.getFixture("xmp/xmp.xmp")); - xmp = StringUtils.trimXMP(xmp); + xmp = Utils.trimXMP(xmp); MapReader reader = new MapReader(xmp); Map elements = reader.readElements(); @@ -26,7 +25,7 @@ void readElements1() throws Exception { @Test void readElements2() throws Exception { String xmp = Files.readString(TestUtil.getFixture("xmp/xmp2.xmp")); - xmp = StringUtils.trimXMP(xmp); + xmp = Utils.trimXMP(xmp); MapReader reader = new MapReader(xmp); Map elements = reader.readElements(); assertEquals(61, elements.size()); diff --git a/src/test/java/edu/illinois/library/cantaloupe/image/xmp/UtilsTest.java b/src/test/java/edu/illinois/library/cantaloupe/image/xmp/UtilsTest.java index 62dbd11c2..386ced649 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/image/xmp/UtilsTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/image/xmp/UtilsTest.java @@ -17,4 +17,32 @@ void testEncapsulateXMP() { assertTrue(actual.endsWith("")); } + /* trimXMP() */ + + @Test + void testTrimXMPWithTrimmableXMP() { + String xmp = "" + + "" + + "" + + "" + + ""; + String result = Utils.trimXMP(xmp); + assertTrue(result.startsWith("")); + } + + @Test + void testTrimXMPWithNonTrimmableXMP() { + String xmp = "" + + ""; + String result = Utils.trimXMP(xmp); + assertSame(xmp, result); + } + + @Test + void testTrimXMPWithNullArgument() { + assertThrows(NullPointerException.class, + () -> Utils.trimXMP(null)); + } + } \ No newline at end of file diff --git a/src/test/java/edu/illinois/library/cantaloupe/util/StringUtilsTest.java b/src/test/java/edu/illinois/library/cantaloupe/util/StringUtilsTest.java index e6b5f00b1..817c9041f 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/util/StringUtilsTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/util/StringUtilsTest.java @@ -218,32 +218,6 @@ void testToByteSizeWithPB() { assertEquals(expected, StringUtils.toByteSize("25 pb")); } - @Test - void testTrimXMPWithTrimmableXMP() { - String xmp = "" + - "" + - "" + - "" + - ""; - String result = StringUtils.trimXMP(xmp); - assertTrue(result.startsWith("")); - } - - @Test - void testTrimXMPWithNonTrimmableXMP() { - String xmp = "" + - ""; - String result = StringUtils.trimXMP(xmp); - assertSame(xmp, result); - } - - @Test - void testTrimXMPWithNullArgument() { - assertThrows(NullPointerException.class, - () -> StringUtils.trimXMP(null)); - } - @Test void testWrap() { String str = "This is a very very very very very very very very long line."; From b5fca99305e746ca03eba3569c5e78dbce642c79 Mon Sep 17 00:00:00 2001 From: Mark Lindeman Date: Fri, 11 Mar 2022 22:52:53 +0100 Subject: [PATCH 025/106] Feature/http proxy support (#553) * Adds support for HTTP Proxies when usinh HttpSource Co-authored-by: mlindeman --- cantaloupe.properties.sample | 5 +++++ .../illinois/library/cantaloupe/config/Key.java | 3 +++ .../library/cantaloupe/source/HttpSource.java | 17 +++++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/cantaloupe.properties.sample b/cantaloupe.properties.sample index 62a9a880e..e5450d649 100644 --- a/cantaloupe.properties.sample +++ b/cantaloupe.properties.sample @@ -212,6 +212,11 @@ HttpSource.chunking.cache.enabled = true # Max per-request chunk cache size. HttpSource.chunking.cache.max_size = 5M +# Enable HTTP Proxy for HttpSource +HttpSource.proxy.http.enabled = false +HttpSource.proxy.http.server = proxy.example.com +HttpSource.proxy.http.port = 8080 + #---------------------------------------- # S3Source #---------------------------------------- diff --git a/src/main/java/edu/illinois/library/cantaloupe/config/Key.java b/src/main/java/edu/illinois/library/cantaloupe/config/Key.java index 977da9d1c..a315294a9 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/config/Key.java +++ b/src/main/java/edu/illinois/library/cantaloupe/config/Key.java @@ -120,6 +120,9 @@ public enum Key { HTTPS_KEY_STORE_PATH("https.key_store_path"), HTTPS_KEY_STORE_TYPE("https.key_store_type"), HTTPS_PORT("https.port"), + HTTPSOURCE_HTTP_PROXY_ENABLED("HttpSource.proxy.http.enabled"), + HTTPSOURCE_HTTP_PROXY_SERVER("HttpSource.proxy.http.server"), + HTTPSOURCE_HTTP_PROXY_PORT("HttpSource.proxy.http.port"), IIIF_1_ENDPOINT_ENABLED("endpoint.iiif.1.enabled"), IIIF_2_ENDPOINT_ENABLED("endpoint.iiif.2.enabled"), IIIF_3_ENDPOINT_ENABLED("endpoint.iiif.3.enabled"), diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java b/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java index 07eb7422e..8edcc20ed 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.net.Proxy; +import java.net.InetSocketAddress; import java.nio.file.AccessDeniedException; import java.nio.file.NoSuchFileException; import java.security.KeyManagementException; @@ -393,6 +395,21 @@ static synchronized OkHttpClient getHTTPClient() { .writeTimeout(getRequestTimeout().getSeconds(), TimeUnit.SECONDS); final Configuration config = Configuration.getInstance(); + + final boolean httpProxyEnabled = config.getBoolean( + Key.HTTPSOURCE_HTTP_PROXY_ENABLED, false); + if (httpProxyEnabled) { + final String httpProxyServer = config.getString(Key.HTTPSOURCE_HTTP_PROXY_SERVER, ""); + if (httpProxyServer == "") { + throw new RuntimeException("proxy server setting HttpSource.proxy.http.server should not be empty"); + } + final int httpProxyPort = config.getInt(Key.HTTPSOURCE_HTTP_PROXY_PORT, 8080); + + LOGGER.trace("Using HTTP Proxy at server {} on port {}", httpProxyServer, httpProxyPort); + Proxy httpProxy = new Proxy(Proxy.Type.HTTP,new InetSocketAddress(httpProxyServer, httpProxyPort)); + builder.proxy(httpProxy); + } + final boolean allowInsecure = config.getBoolean( Key.HTTPSOURCE_ALLOW_INSECURE, false); From 43c42f0101f3db280d05249198e823e8e1a6f3be Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Fri, 11 Mar 2022 16:28:44 -0600 Subject: [PATCH 026/106] Fix failing tests --- .../cantaloupe/cache/S3MultipartAsyncOutputStreamTest.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStreamTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStreamTest.java index 721ff18f9..da3a1d60b 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStreamTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStreamTest.java @@ -118,6 +118,7 @@ void closeMarksInstanceComplete() throws Exception { byte[] bytes = new byte[1024 * 1024]; new SecureRandom().nextBytes(bytes); instance.write(bytes); + instance.setComplete(true); instance.close(); //noinspection SynchronizationOnLocalVariableOrMethodParameter @@ -146,6 +147,7 @@ void write1WithMultipleParts() throws Exception { for (byte b : expectedBytes) { instance.write(b); } + instance.setComplete(true); instance.close(); //noinspection SynchronizationOnLocalVariableOrMethodParameter @@ -174,6 +176,7 @@ void write1WithSinglePart() throws Exception { for (byte b : expectedBytes) { instance.write(b); } + instance.setComplete(true); instance.close(); //noinspection SynchronizationOnLocalVariableOrMethodParameter @@ -205,6 +208,7 @@ void write2WithMultipleParts() throws Exception { try { instance.write(expectedBytes); + instance.setComplete(true); instance.close(); //noinspection SynchronizationOnLocalVariableOrMethodParameter @@ -234,6 +238,7 @@ void write2WithSinglePart() throws Exception { byte[] expectedBytes = new byte[1024 * 1024]; // smaller than part size new SecureRandom().nextBytes(expectedBytes); instance.write(expectedBytes); + instance.setComplete(true); instance.close(); //noinspection SynchronizationOnLocalVariableOrMethodParameter @@ -265,6 +270,7 @@ void write3WithMultipleParts() throws Exception { try { instance.write(expectedBytes, 0, expectedBytes.length); + instance.setComplete(true); instance.close(); //noinspection SynchronizationOnLocalVariableOrMethodParameter @@ -294,6 +300,7 @@ void write3WithSinglePart() throws Exception { byte[] expectedBytes = new byte[1024 * 1024]; // smaller than part size new SecureRandom().nextBytes(expectedBytes); instance.write(expectedBytes, 0, expectedBytes.length); + instance.setComplete(true); instance.close(); //noinspection SynchronizationOnLocalVariableOrMethodParameter From 268d7bdcf31d7ccba134a16c7059b3ffbc6c1021 Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Fri, 18 Mar 2022 09:18:38 -0500 Subject: [PATCH 027/106] Improve documentation --- .../cache/S3MultipartAsyncOutputStream.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStream.java b/src/main/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStream.java index 00b93b54e..56990bf74 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStream.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStream.java @@ -37,12 +37,20 @@ * After {@link #close()} returns, the resulting object will take a little bit * of time to appear in the bucket.

      * + *

      Note that because this is a {@link CompletableOutputStream}, if the + * instance is not {@link #setComplete(boolean) marked as complete} before + * closure, the upload will be aborted.

      + * *

      Multi-part uploads can reduce memory usage when uploading objects larger * than the part length, as that is roughly the maximum amount that has to be * buffered in memory (provided that the length of the byte array passed to * either of the {@link #write} methods is not greater than the part * length).

      * + *

      N.B.: Incomplete uploads should be aborted automatically, but when using + * Amazon S3, it may be helpful to enable the {@literal + * AbortIncompleteMultipartUpload} lifecycle rule as a fallback.

      + * * @author Alex Dolski UIUC * @since 6.0 */ @@ -59,10 +67,6 @@ void add(Runnable task) { workQueue.add(task); } - void stop() { - isStopped = true; - } - @Override public void run() { while (!isDone && !isStopped) { From 4356503a339d61911ec794ac91b5fa5d95b1a9ba Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Fri, 18 Mar 2022 10:30:31 -0500 Subject: [PATCH 028/106] Add a serializationTimestamp key to infos (#571) --- .../library/cantaloupe/image/Info.java | 61 +++++++-- .../cantaloupe/image/InfoDeserializer.java | 14 +- .../cantaloupe/image/InfoSerializer.java | 34 +++-- .../library/cantaloupe/image/InfoTest.java | 123 ++++++++++++++++-- 4 files changed, 193 insertions(+), 39 deletions(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/Info.java b/src/main/java/edu/illinois/library/cantaloupe/image/Info.java index 12682aee2..6656c3fdd 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/image/Info.java +++ b/src/main/java/edu/illinois/library/cantaloupe/image/Info.java @@ -17,6 +17,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Path; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -205,11 +206,19 @@ public enum Serialization { * *

      Introduced in application version 5.0.

      */ - VERSION_4(4); + VERSION_4(4), + + /** + *

      Added a {@code serializationTimestamp} key containing an ISO 8601 + * timestamp in UTC.

      + * + *

      Introduced in application version 6.0.

      + */ + VERSION_5(5); private final int version; - static final Serialization CURRENT = VERSION_4; + static final Serialization CURRENT = VERSION_5; Serialization(int version) { this.version = version; @@ -220,11 +229,12 @@ int getVersion() { } } - private String appVersion = Application.getVersion(); + private String appVersion = Application.getVersion(); private Identifier identifier; private MediaType mediaType; - private Metadata metadata = new Metadata(); - private Serialization serialization = Serialization.CURRENT; + private Metadata metadata = new Metadata(); + private Serialization serialization = Serialization.CURRENT; + private Instant serializationTimestamp; /** * Ordered list of subimages. The main image is at index {@code 0}. @@ -268,19 +278,24 @@ public Info() { images.add(new Image()); } + /** + * N.B.: the {@link #getSerializationTimestamp() serialization timestamp} + * is not considered. + */ @Override public boolean equals(Object obj) { if (obj == this) { return true; } else if (obj instanceof Info) { Info other = (Info) obj; - return Objects.equals(other.getApplicationVersion(), getApplicationVersion()) && - Objects.equals(other.getSerialization(), getSerialization()) && - Objects.equals(other.getIdentifier(), getIdentifier()) && - Objects.equals(other.getMetadata(), getMetadata()) && - Objects.equals(other.getSourceFormat(), getSourceFormat()) && - other.getNumResolutions() == getNumResolutions() && - other.getImages().equals(getImages()); + boolean a = Objects.equals(other.getApplicationVersion(), getApplicationVersion()); + boolean b = Objects.equals(other.getSerialization(), getSerialization()); + boolean c = Objects.equals(other.getIdentifier(), getIdentifier()); + boolean d = Objects.equals(other.getMetadata(), getMetadata()); + boolean e = Objects.equals(other.getSourceFormat(), getSourceFormat()); + boolean f = other.getNumResolutions() == getNumResolutions(); + boolean g = other.getImages().equals(getImages()); + return a && b && c && d && e && f && g; } return super.equals(obj); } @@ -367,6 +382,20 @@ public Serialization getSerialization() { return serialization; } + /** + * N.B.: Although it would be possible for most {@link + * edu.illinois.library.cantaloupe.cache.DerivativeCache}s to obtain a + * serialization timestamp by other means, such as e.g. filesystem + * attributes, storing it within the serialized instance makes a separate + * I/O call unnecessary. + * + * @return Timestamp that the instance was serialized. + * @since 6.0 + */ + public Instant getSerializationTimestamp() { + return serializationTimestamp; + } + /** * @return Size of the main image. */ @@ -484,6 +513,14 @@ public void setPersistable(boolean isComplete) { this.isPersistable = isComplete; } + /** + * @param timestamp Time at which the instance is serialized. + * @since 6.0 + */ + void setSerializationTimestamp(Instant timestamp) { + this.serializationTimestamp = timestamp; + } + /** * @param version One of the {@link Serialization} versions. This value is * not serialized (the current serialization version is diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/InfoDeserializer.java b/src/main/java/edu/illinois/library/cantaloupe/image/InfoDeserializer.java index 0800eee0a..e88570eb7 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/image/InfoDeserializer.java +++ b/src/main/java/edu/illinois/library/cantaloupe/image/InfoDeserializer.java @@ -8,6 +8,7 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.time.Instant; import static edu.illinois.library.cantaloupe.image.InfoSerializer.APPLICATION_VERSION_KEY; import static edu.illinois.library.cantaloupe.image.InfoSerializer.IDENTIFIER_KEY; @@ -15,6 +16,7 @@ import static edu.illinois.library.cantaloupe.image.InfoSerializer.MEDIA_TYPE_KEY; import static edu.illinois.library.cantaloupe.image.InfoSerializer.METADATA_KEY; import static edu.illinois.library.cantaloupe.image.InfoSerializer.NUM_RESOLUTIONS_KEY; +import static edu.illinois.library.cantaloupe.image.InfoSerializer.SERIALIZATION_TIMESTAMP_KEY; import static edu.illinois.library.cantaloupe.image.InfoSerializer.SERIALIZATION_VERSION_KEY; /** @@ -28,10 +30,16 @@ final class InfoDeserializer extends JsonDeserializer { public Info deserialize(JsonParser parser, DeserializationContext deserializationContext) throws IOException { // N.B.: keys may or may not exist in different serializations, - // documented inline. Even for keys that are supposed to always exist, - // we have to check for them anyway because they may not exist in tests. - final Info info = new Info(); + // documented inline. Even keys that are supposed to always exist may + // not exist in tests, so we have to check for them anyway. + final Info info = new Info(); final JsonNode node = parser.getCodec().readTree(parser); + { // serializationTimestamp (does not exist in < 6.0 serializations) + JsonNode timestampNode = node.get(SERIALIZATION_TIMESTAMP_KEY); + if (timestampNode != null) { + info.setSerializationTimestamp(Instant.parse(timestampNode.textValue())); + } + } { // applicationVersion (does not exist in < 5.0 serializations) JsonNode appVersionNode = node.get(APPLICATION_VERSION_KEY); if (appVersionNode != null) { diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/InfoSerializer.java b/src/main/java/edu/illinois/library/cantaloupe/image/InfoSerializer.java index ba70a4499..f337ee27a 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/image/InfoSerializer.java +++ b/src/main/java/edu/illinois/library/cantaloupe/image/InfoSerializer.java @@ -7,6 +7,7 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.time.Instant; /** * Serializes an {@link Info}. @@ -15,13 +16,14 @@ */ final class InfoSerializer extends JsonSerializer { - static final String APPLICATION_VERSION_KEY = "applicationVersion"; - static final String IDENTIFIER_KEY = "identifier"; - static final String IMAGES_KEY = "images"; - static final String MEDIA_TYPE_KEY = "mediaType"; - static final String METADATA_KEY = "metadata"; - static final String NUM_RESOLUTIONS_KEY = "numResolutions"; - static final String SERIALIZATION_VERSION_KEY = "serializationVersion"; + static final String APPLICATION_VERSION_KEY = "applicationVersion"; + static final String IDENTIFIER_KEY = "identifier"; + static final String IMAGES_KEY = "images"; + static final String MEDIA_TYPE_KEY = "mediaType"; + static final String METADATA_KEY = "metadata"; + static final String NUM_RESOLUTIONS_KEY = "numResolutions"; + static final String SERIALIZATION_TIMESTAMP_KEY = "serializationTimestamp"; + static final String SERIALIZATION_VERSION_KEY = "serializationVersion"; @Override public void serialize(Info info, @@ -29,19 +31,27 @@ public void serialize(Info info, SerializerProvider serializerProvider) throws IOException { generator.writeStartObject(); // application version - generator.writeStringField(APPLICATION_VERSION_KEY, Application.getVersion()); + generator.writeStringField(APPLICATION_VERSION_KEY, + Application.getVersion()); // serialization version - generator.writeNumberField(SERIALIZATION_VERSION_KEY, Info.Serialization.CURRENT.getVersion()); + generator.writeNumberField(SERIALIZATION_VERSION_KEY, + Info.Serialization.CURRENT.getVersion()); + // serialization timestamp + generator.writeStringField(SERIALIZATION_TIMESTAMP_KEY, + Instant.now().toString()); // identifier if (info.getIdentifier() != null) { - generator.writeStringField(IDENTIFIER_KEY, info.getIdentifier().toString()); + generator.writeStringField(IDENTIFIER_KEY, + info.getIdentifier().toString()); } // mediaType if (info.getMediaType() != null) { - generator.writeStringField(MEDIA_TYPE_KEY, info.getMediaType().toString()); + generator.writeStringField(MEDIA_TYPE_KEY, + info.getMediaType().toString()); } // numResolutions - generator.writeNumberField(NUM_RESOLUTIONS_KEY, info.getNumResolutions()); + generator.writeNumberField(NUM_RESOLUTIONS_KEY, + info.getNumResolutions()); // images generator.writeArrayFieldStart(IMAGES_KEY); info.getImages().forEach(image -> { diff --git a/src/test/java/edu/illinois/library/cantaloupe/image/InfoTest.java b/src/test/java/edu/illinois/library/cantaloupe/image/InfoTest.java index c724ae6e9..58c8785e9 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/image/InfoTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/image/InfoTest.java @@ -12,6 +12,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Instant; import static org.junit.jupiter.api.Assertions.*; @@ -207,7 +208,8 @@ void testFromJSONWithPath() throws Exception { Files.write(tempFile, json.getBytes(StandardCharsets.UTF_8)); Info info = Info.fromJSON(tempFile); - assertEquals(info.toString(), instance.toString()); + assertEquals(obscureTimestamps(info.toString()), + obscureTimestamps(instance.toString())); } finally { if (tempFile != null) { Files.deleteIfExists(tempFile); @@ -223,7 +225,8 @@ void testFromJSONWithInputStream() throws Exception { InputStream inputStream = new ByteArrayInputStream(json.getBytes()); Info info = Info.fromJSON(inputStream); - assertEquals(info.toString(), instance.toString()); + assertEquals(obscureTimestamps(info.toString()), + obscureTimestamps(instance.toString())); } /* fromJSON(String) */ @@ -232,14 +235,15 @@ void testFromJSONWithInputStream() throws Exception { void testFromJSONWithString() throws Exception { String json = instance.toJSON(); Info info = Info.fromJSON(json); - assertEquals(info.toString(), instance.toString()); + assertEquals(obscureTimestamps(info.toString()), + obscureTimestamps(instance.toString())); } /* fromJSON() serialization */ @Test void testFromJSONWithVersion2Serialization() throws Exception { - String v34json = "{\n" + + String v2json = "{\n" + " \"mediaType\": \"image/jpeg\",\n" + " \"images\": [\n" + " {\n" + @@ -250,7 +254,7 @@ void testFromJSONWithVersion2Serialization() throws Exception { " }\n" + " ]\n" + "}"; - Info actual = Info.fromJSON(v34json); + Info actual = Info.fromJSON(v2json); Info expected = Info.builder() .withFormat(Format.get("jpg")) .withSize(100, 80) @@ -261,7 +265,7 @@ void testFromJSONWithVersion2Serialization() throws Exception { @Test void testFromJSONWithVersion3Serialization() throws Exception { - String v4json = "{\n" + + String v3json = "{\n" + " \"identifier\": \"cats\",\n" + " \"mediaType\": \"image/jpeg\",\n" + " \"numResolutions\": 3,\n" + @@ -275,7 +279,7 @@ void testFromJSONWithVersion3Serialization() throws Exception { " }\n" + " ]\n" + "}"; - Info actual = Info.fromJSON(v4json); + Info actual = Info.fromJSON(v3json); Info expected = Info.builder() .withIdentifier(new Identifier("cats")) .withFormat(Format.get("jpg")) @@ -288,7 +292,7 @@ void testFromJSONWithVersion3Serialization() throws Exception { @Test void testFromJSONWithVersion4Serialization() throws Exception { - String v5json = "{\n" + + String v4json = "{\n" + " \"applicationVersion\": \"5.0\",\n" + " \"serializationVersion\": 4,\n" + " \"identifier\": \"cats\",\n" + @@ -306,9 +310,10 @@ void testFromJSONWithVersion4Serialization() throws Exception { " \"xmp\": \"\"\n" + " }\n" + "}"; + Info actual = Info.fromJSON(v4json); + Metadata metadata = new Metadata(); metadata.setXMP(""); - Info actual = Info.fromJSON(v5json); Info expected = Info.builder() .withIdentifier(new Identifier("cats")) .withFormat(Format.get("jpg")) @@ -318,6 +323,45 @@ void testFromJSONWithVersion4Serialization() throws Exception { .withTileSize(50, 40) .build(); expected.setApplicationVersion("5.0"); + actual.setSerializationVersion(Info.Serialization.CURRENT.getVersion()); + assertEquals(expected, actual); + } + + @Test + void testFromJSONWithVersion5Serialization() throws Exception { + Instant timestamp = Instant.now(); + String v6json = "{\n" + + " \"applicationVersion\": \"6.0\",\n" + + " \"serializationVersion\": 5,\n" + + " \"serializationTimestamp\": \"" + timestamp.toString() + "\",\n" + + " \"identifier\": \"cats\",\n" + + " \"mediaType\": \"image/jpeg\",\n" + + " \"numResolutions\": 3,\n" + + " \"images\": [\n" + + " {\n" + + " \"width\": 100,\n" + + " \"height\": 80,\n" + + " \"tileWidth\": 50,\n" + + " \"tileHeight\": 40\n" + + " }\n" + + " ],\n" + + " \"metadata\": {\n" + + " \"xmp\": \"\"\n" + + " }\n" + + "}"; + Metadata metadata = new Metadata(); + metadata.setXMP(""); + Info actual = Info.fromJSON(v6json); + Info expected = Info.builder() + .withIdentifier(new Identifier("cats")) + .withFormat(Format.get("jpg")) + .withNumResolutions(3) + .withMetadata(metadata) + .withSize(100, 80) + .withTileSize(50, 40) + .build(); + expected.setApplicationVersion("6.0"); + expected.setSerializationTimestamp(timestamp); assertEquals(expected, actual); } @@ -373,6 +417,20 @@ void testEqualsWithDifferentSerializationVersions() { assertNotEquals(instance, info2); } + @Test + void testEqualsWithDifferentSerializationTimestamps() { + Info info2 = Info.builder() + .withIdentifier(instance.getIdentifier()) + .withSize(instance.getSize()) + .withTileSize(instance.getImages().get(0).getTileSize()) + .withFormat(instance.getSourceFormat()) + .withNumResolutions(instance.getNumResolutions()) + .withMetadata(instance.getMetadata()) + .build(); + info2.setSerializationTimestamp(Instant.now()); + assertEquals(instance, info2); + } + @Test void testEqualsWithDifferentIdentifiers() { Info info2 = Info.builder() @@ -633,6 +691,23 @@ void testHashCodeWithEqualInstances() { assertEquals(instance.hashCode(), info2.hashCode()); } + @Test + void testHashCodeWithDifferentSerializationTimestamps() { + Metadata metadata2 = new Metadata(); + metadata2.setXMP(""); + + Info info2 = Info.builder() + .withIdentifier(instance.getIdentifier()) + .withSize(instance.getSize()) + .withTileSize(instance.getImages().get(0).getTileSize()) + .withFormat(instance.getSourceFormat()) + .withNumResolutions(instance.getNumResolutions()) + .withMetadata(metadata2) + .build(); + info2.setSerializationTimestamp(Instant.now()); + assertEquals(instance.hashCode(), info2.hashCode()); + } + @Test void testHashCodeWithDifferentIdentifiers() { Info info2 = Info.builder() @@ -859,6 +934,15 @@ void testSetPersistable() { assertFalse(instance.isPersistable()); } + /* setSerializationTimestamp() */ + + @Test + void testSetSerializationTimestamp() { + Instant timestamp = Instant.now(); + instance.setSerializationTimestamp(timestamp); + assertEquals(timestamp, instance.getSerializationTimestamp()); + } + /* setSerializationVersion() */ @Test @@ -889,6 +973,7 @@ void testToJSONContents() throws Exception { assertEquals("{" + "\"applicationVersion\":\"" + Application.getVersion() + "\"," + "\"serializationVersion\":" + Info.Serialization.CURRENT.getVersion() + "," + + "\"serializationTimestamp\":\"0000-00-00T00:00:00.000000Z\"," + "\"identifier\":\"cats\"," + "\"mediaType\":\"image/jpeg\"," + "\"numResolutions\":3," + @@ -903,7 +988,7 @@ void testToJSONContents() throws Exception { "\"xmp\":\"\"" + "}" + "}", - instance.toJSON()); + obscureTimestamps(instance.toJSON())); } @Test @@ -923,7 +1008,8 @@ void testToJSONOmitsNullValues() throws Exception { @Test void testToString() throws Exception { - assertEquals(instance.toJSON(), instance.toString()); + assertEquals(obscureTimestamps(instance.toJSON()), + obscureTimestamps(instance.toString())); } /* writeAsJSON() */ @@ -932,7 +1018,20 @@ void testToString() throws Exception { void testWriteAsJSON() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); instance.writeAsJSON(baos); - assertArrayEquals(baos.toByteArray(), instance.toJSON().getBytes()); + + String expected = baos.toString(StandardCharsets.UTF_8); + String actual = new String(instance.toJSON().getBytes(), + StandardCharsets.UTF_8); + assertEquals(obscureTimestamps(expected), obscureTimestamps(actual)); + } + + /** + * Converts any ISO-8601 timestamps in the given string to + * {@literal 0000-00-00T00:00:00.000000Z}. + */ + private static String obscureTimestamps(String inString) { + return inString.replaceAll("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d+Z", + "0000-00-00T00:00:00.000000Z"); } } From 772a3f4c9ac645e3c6b8b0da9d277ee61b6277fc Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Fri, 18 Mar 2022 10:30:55 -0500 Subject: [PATCH 029/106] Responses for cached infos include a Last-Modified header (#571) --- CHANGES.md | 1 + .../resource/iiif/v1/InformationResource.java | 21 +++++++++++++++---- .../resource/iiif/v2/InformationResource.java | 18 +++++++++++++--- .../resource/iiif/v3/InformationResource.java | 18 +++++++++++++--- .../resource/iiif/ImageAPIResourceTester.java | 16 ++++++++++++++ .../iiif/v1/InformationResourceTest.java | 9 ++++++++ .../iiif/v2/InformationResourceTest.java | 9 ++++++++ .../iiif/v3/InformationResourceTest.java | 9 ++++++++ 8 files changed, 91 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 151cdca4c..d3e9c6444 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ # 6.0 +* Responses for cached infos include a `Last-Modified` header. * The health endpoint is enabled via `endpoint.health.enabled` rather than `endpoint.api.enabled`. * HttpSource can be configured to send a ranged GET request instead of a HEAD diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResource.java index 360566822..3e6903cf7 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResource.java @@ -1,11 +1,14 @@ package edu.illinois.library.cantaloupe.resource.iiif.v1; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Set; import edu.illinois.library.cantaloupe.http.Method; -import edu.illinois.library.cantaloupe.http.Status; import edu.illinois.library.cantaloupe.image.Format; import edu.illinois.library.cantaloupe.image.Info; import edu.illinois.library.cantaloupe.resource.JacksonRepresentation; @@ -86,16 +89,26 @@ public void knowAvailableOutputFormats(Set formats) { getPageIndex(), getMetaIdentifier().getScaleConstraint()); - addHeaders(iiifInfo); + addHeaders(info, iiifInfo); new JacksonRepresentation(iiifInfo) .write(getResponse().getOutputStream()); } } - private void addHeaders(Information info) { + private void addHeaders(Info info, Information iiifInfo) { + // Content-Type getResponse().setHeader("Content-Type", getNegotiatedMediaType()); + // Link getResponse().setHeader("Link", - String.format("<%s>;rel=\"profile\";", info.profile)); + String.format("<%s>;rel=\"profile\";", iiifInfo.profile)); + // Last-Modified + if (info.getSerializationTimestamp() != null) { + getResponse().setHeader("Last-Modified", + DateTimeFormatter.RFC_1123_DATE_TIME + .withLocale(Locale.UK) + .withZone(ZoneId.systemDefault()) + .format(info.getSerializationTimestamp())); + } } /** diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResource.java index 138ddf34c..daea7fa32 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResource.java @@ -1,11 +1,14 @@ package edu.illinois.library.cantaloupe.resource.iiif.v2; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Set; import edu.illinois.library.cantaloupe.http.Method; -import edu.illinois.library.cantaloupe.http.Status; import edu.illinois.library.cantaloupe.image.Format; import edu.illinois.library.cantaloupe.image.Info; import edu.illinois.library.cantaloupe.processor.codec.ImageWriterFactory; @@ -83,14 +86,23 @@ public void knowAvailableOutputFormats(Set formats) { .withCallback(new CustomCallback()) .build()) { Info info = handler.handle(); - addHeaders(); + addHeaders(info); newRepresentation(info, availableOutputFormats) .write(getResponse().getOutputStream()); } } - private void addHeaders() { + private void addHeaders(Info info) { + // Content-Type getResponse().setHeader("Content-Type", getNegotiatedMediaType()); + // Last-Modified + if (info.getSerializationTimestamp() != null) { + getResponse().setHeader("Last-Modified", + DateTimeFormatter.RFC_1123_DATE_TIME + .withLocale(Locale.UK) + .withZone(ZoneId.systemDefault()) + .format(info.getSerializationTimestamp())); + } } /** diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResource.java index 53106fcb5..fa5db95d7 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResource.java @@ -1,11 +1,14 @@ package edu.illinois.library.cantaloupe.resource.iiif.v3; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Set; import edu.illinois.library.cantaloupe.http.Method; -import edu.illinois.library.cantaloupe.http.Status; import edu.illinois.library.cantaloupe.image.Format; import edu.illinois.library.cantaloupe.image.Info; import edu.illinois.library.cantaloupe.processor.codec.ImageWriterFactory; @@ -83,14 +86,23 @@ public void knowAvailableOutputFormats(Set formats) { .withCallback(new CustomCallback()) .build()) { Info info = handler.handle(); - addHeaders(); + addHeaders(info); newRepresentation(info, availableOutputFormats) .write(getResponse().getOutputStream()); } } - private void addHeaders() { + private void addHeaders(Info info) { + // Content-Type getResponse().setHeader("Content-Type", getNegotiatedContentType()); + // Last-Modified + if (info.getSerializationTimestamp() != null) { + getResponse().setHeader("Last-Modified", + DateTimeFormatter.RFC_1123_DATE_TIME + .withLocale(Locale.UK) + .withZone(ZoneId.systemDefault()) + .format(info.getSerializationTimestamp())); + } } /** diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/ImageAPIResourceTester.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/ImageAPIResourceTester.java index ef04a9365..abdc24667 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/ImageAPIResourceTester.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/ImageAPIResourceTester.java @@ -290,6 +290,22 @@ public void testForbidden(URI uri) { assertStatus(403, uri); } + public void testLastModifiedHeaderWhenDerivativeCacheIsEnabled(URI uri) + throws Exception { + initializeFilesystemCache(); + + Client client = newClient(uri); + try { + // request a resource once to cache it + client.send(); + // request it again to get the Last-Modified header + Response response = client.send(); + assertNotNull(response.getHeaders().getFirstValue("Last-Modified")); + } finally { + client.stop(); + } + } + public void testNotFound(URI uri) { assertStatus(404, uri); } diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResourceTest.java index 9c992a266..a78c4db13 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResourceTest.java @@ -552,6 +552,8 @@ void testGETResponseHeaders() throws Exception { headers.getFirstValue("Content-Type"))); // Date assertNotNull(headers.getFirstValue("Date")); + // Last-Modified (only present for cached infos, tested separately) + assertNull(headers.getFirstValue("Last-Modified")); // Link assertTrue(headers.getFirstValue("Link").contains("://")); // Server @@ -570,6 +572,13 @@ void testGETResponseHeaders() throws Exception { headers.getFirstValue("X-Powered-By")); } + @Test + void testGETLastModifiedResponseHeaderWhenDerivativeCacheIsEnabled() + throws Exception { + URI uri = getHTTPURI("/" + IMAGE + "/info.json"); + tester.testLastModifiedHeaderWhenDerivativeCacheIsEnabled(uri); + } + @Test void testOPTIONSWhenEnabled() throws Exception { Configuration config = Configuration.getInstance(); diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResourceTest.java index b097e6167..d2b11caed 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResourceTest.java @@ -567,6 +567,8 @@ void testGETResponseHeaders() throws Exception { headers.getFirstValue("Content-Type"))); // Date assertNotNull(headers.getFirstValue("Date")); + // Last-Modified (only present for cached infos, tested separately) + assertNull(headers.getFirstValue("Last-Modified")); // Server assertNotNull(headers.getFirstValue("Server")); // Vary @@ -583,6 +585,13 @@ void testGETResponseHeaders() throws Exception { headers.getFirstValue("X-Powered-By")); } + @Test + void testGETLastModifiedResponseHeaderWhenDerivativeCacheIsEnabled() + throws Exception { + URI uri = getHTTPURI("/" + IMAGE + "/info.json"); + tester.testLastModifiedHeaderWhenDerivativeCacheIsEnabled(uri); + } + @Test void testOPTIONSWhenEnabled() throws Exception { Configuration config = Configuration.getInstance(); diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResourceTest.java index bec485560..8618c03cb 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResourceTest.java @@ -542,6 +542,8 @@ void testGETResponseHeaders() throws Exception { headers.getFirstValue("Content-Type"))); // Date assertNotNull(headers.getFirstValue("Date")); + // Last-Modified (only present for cached infos, tested separately) + assertNull(headers.getFirstValue("Last-Modified")); // Server assertNotNull(headers.getFirstValue("Server")); // Vary @@ -558,6 +560,13 @@ void testGETResponseHeaders() throws Exception { headers.getFirstValue("X-Powered-By")); } + @Test + void testGETLastModifiedResponseHeaderWhenDerivativeCacheIsEnabled() + throws Exception { + URI uri = getHTTPURI("/" + IMAGE + "/info.json"); + tester.testLastModifiedHeaderWhenDerivativeCacheIsEnabled(uri); + } + @Test void testOPTIONSWhenEnabled() throws Exception { Configuration config = Configuration.getInstance(); From cdbae5b2fcc13a9c2e6cfc6c28cfd4833c2bc44b Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Fri, 18 Mar 2022 11:13:22 -0500 Subject: [PATCH 030/106] DerivativeCache.getInfo() populates the returned instance's serialization timestamp if null (#571) --- CHANGES.md | 4 +++- .../cantaloupe/cache/AzureStorageCache.java | 8 +++++++- .../cantaloupe/cache/DerivativeCache.java | 7 +++++++ .../cantaloupe/cache/FilesystemCache.java | 17 ++++++++++++----- .../library/cantaloupe/cache/S3Cache.java | 5 +++++ .../library/cantaloupe/image/Info.java | 2 +- .../cantaloupe/cache/AbstractCacheTest.java | 18 ++++++++++++++++++ 7 files changed, 53 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d3e9c6444..12e85f2be 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,9 @@ # 6.0 -* Responses for cached infos include a `Last-Modified` header. +* Responses for infos serialized in this version or later by any cache, or in + earlier versions by AzureStorageCache, FilesystemCache, or S3Cache, include a + `Last-Modified` header. * The health endpoint is enabled via `endpoint.health.enabled` rather than `endpoint.api.enabled`. * HttpSource can be configured to send a ranged GET request instead of a HEAD diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java index 1ba50b610..2d212dd6b 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/AzureStorageCache.java @@ -199,7 +199,7 @@ private static Instant getEarliestValidInstant() { @Override public Optional getInfo(Identifier identifier) throws IOException { - final String containerName = getContainerName(); + final String containerName = getContainerName(); final CloudBlobClient client = getClientInstance(); try { @@ -213,6 +213,12 @@ public Optional getInfo(Identifier identifier) throws IOException { if (isValid(blob)) { try (InputStream is = blob.openInputStream()) { Info info = Info.fromJSON(is); + // Populate the serialization timestamp if it is not + // already, as suggested by the method contract. + if (info.getSerializationTimestamp() == null) { + info.setSerializationTimestamp( + blob.getProperties().getLastModified().toInstant()); + } LOGGER.debug("getInfo(): read {} from container {} in {}", objectKey, containerName, watch); return Optional.of(info); diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java index cd03641f8..f7cdd0032 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/DerivativeCache.java @@ -20,6 +20,13 @@ public interface DerivativeCache extends Cache { *

      Reads the cached image information corresponding to the given * identifier.

      * + *

      If the returned instance's {@link Info#getSerializationTimestamp() + * serialization timestamp} is {@code null} (which it will be for {@link + * Info.Serialization serialization versions} earlier than {@link + * Info.Serialization#VERSION_5}), implementations should try to + * populate it with the last-modified time of the cached resource, if + * possible.

      + * *

      If invalid image information exists in the cache, implementations * should delete it—ideally asynchronously.

      * diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java index d0defe4d2..ab0dc383a 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/FilesystemCache.java @@ -17,7 +17,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.file.FileAlreadyExistsException; import java.nio.file.FileVisitOption; import java.nio.file.Files; @@ -126,8 +126,8 @@ private static class ConcurrentFileOutputStream private boolean isClosed = false; private final Object lock; private final Path tempFile; - private T toRemove; - private OutputStream wrappedOutputStream; + private final T toRemove; + private final OutputStream wrappedOutputStream; /** * @param tempFile Pathname of the temp file to write to. @@ -316,7 +316,7 @@ static String hashedPathFragment(String uniqueString) { try { final MessageDigest digest = MessageDigest.getInstance(HASH_ALGORITHM); - digest.update(uniqueString.getBytes(Charset.forName("UTF8"))); + digest.update(uniqueString.getBytes(StandardCharsets.UTF_8)); final String sum = Hex.encodeHexString(digest.digest()); final Configuration config = Configuration.getInstance(); @@ -526,7 +526,14 @@ public Optional getInfo(Identifier identifier) throws IOException { final Path cacheFile = infoFile(identifier); if (!isExpired(cacheFile)) { LOGGER.debug("getInfo(): hit: {}", cacheFile); - return Optional.of(Info.fromJSON(cacheFile)); + Info info = Info.fromJSON(cacheFile); + // Populate the serialization timestamp if it is not + // already, as suggested by the method contract. + if (info.getSerializationTimestamp() == null) { + info.setSerializationTimestamp( + Files.getLastModifiedTime(cacheFile).toInstant()); + } + return Optional.of(info); } else { purgeAsync(cacheFile); } diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java index b453a2850..d01117de1 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java @@ -135,6 +135,11 @@ public Optional getInfo(Identifier identifier) throws IOException { // This extra validity check may be needed with minio server if (is != null && is.response().lastModified().isAfter(earliestValidInstant())) { final Info info = Info.fromJSON(is); + // Populate the serialization timestamp if it is not already, + // as suggested by the method contract. + if (info.getSerializationTimestamp() == null) { + info.setSerializationTimestamp(is.response().lastModified()); + } LOGGER.debug("getInfo(): read {} from bucket {} in {}", objectKey, bucketName, watch); touchAsync(objectKey); diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/Info.java b/src/main/java/edu/illinois/library/cantaloupe/image/Info.java index 6656c3fdd..522f3c10e 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/image/Info.java +++ b/src/main/java/edu/illinois/library/cantaloupe/image/Info.java @@ -517,7 +517,7 @@ public void setPersistable(boolean isComplete) { * @param timestamp Time at which the instance is serialized. * @since 6.0 */ - void setSerializationTimestamp(Instant timestamp) { + public void setSerializationTimestamp(Instant timestamp) { this.serializationTimestamp = timestamp; } diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java index 586453de5..f2260977b 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java @@ -21,6 +21,7 @@ import java.util.Optional; import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assumptions.*; abstract class AbstractCacheTest extends BaseTest { @@ -88,6 +89,23 @@ void testGetInfoWithNonexistentImage() throws Exception { assertFalse(instance.getInfo(new Identifier("bogus")).isPresent()); } + @Test + void testGetInfoPopulatesSerializationTimestampWhenNotAlreadySet() + throws Exception { + // These caches don't support this feature. + assumeFalse(this instanceof HeapCacheTest || + this instanceof JdbcCacheTest || + this instanceof RedisCacheTest); + final DerivativeCache instance = newInstance(); + + Identifier identifier = new Identifier("cats"); + Info info = new Info(); + instance.put(identifier, info); + + info = instance.getInfo(identifier).get(); + assertNotNull(info.getSerializationTimestamp()); + } + @Test void testGetInfoConcurrently() { // This is tested in testPutConcurrently() From b1d74fa38f44e6438c04da5a2482e53241183e4d Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Mon, 21 Mar 2022 09:30:28 -0500 Subject: [PATCH 031/106] Redesign Source.checkAccess() as stat() (#571) --- UPGRADING.md | 5 + delegates.rb.sample | 27 ++++-- .../cantaloupe/delegate/DelegateMethod.java | 7 ++ .../cantaloupe/delegate/DelegateProxy.java | 7 ++ .../delegate/JRubyDelegateProxy.java | 10 ++ .../cantaloupe/delegate/JavaDelegate.java | 25 ++++- .../delegate/JavaDelegateProxy.java | 8 ++ .../resource/ImageRequestHandler.java | 3 +- .../resource/InformationRequestHandler.java | 3 +- .../cantaloupe/source/AzureStorageSource.java | 8 +- .../cantaloupe/source/FilesystemSource.java | 8 +- .../library/cantaloupe/source/HttpSource.java | 24 ++++- .../library/cantaloupe/source/JdbcSource.java | 61 ++++++++---- .../library/cantaloupe/source/S3Source.java | 15 ++- .../library/cantaloupe/source/Source.java | 15 +-- .../library/cantaloupe/source/StatResult.java | 20 ++++ .../delegate/JRubyDelegateProxyTest.java | 8 ++ .../resource/iiif/ImageAPIResourceTester.java | 7 +- .../cantaloupe/source/AbstractSourceTest.java | 96 +++++++++++-------- .../cantaloupe/source/AccessDeniedSource.java | 2 +- .../source/AzureStorageSourceTest.java | 6 +- .../source/FilesystemSourceTest.java | 8 +- .../cantaloupe/source/HttpSourceTest.java | 24 ++--- .../cantaloupe/source/JdbcSourceTest.java | 41 ++++---- .../cantaloupe/source/MockFileSource.java | 3 +- .../cantaloupe/source/MockStreamSource.java | 3 +- .../cantaloupe/source/S3SourceTest.java | 8 +- src/test/resources/delegates.rb | 4 + 28 files changed, 323 insertions(+), 133 deletions(-) create mode 100644 src/main/java/edu/illinois/library/cantaloupe/source/StatResult.java diff --git a/UPGRADING.md b/UPGRADING.md index fd790a612..58261b77b 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -9,6 +9,11 @@ current version. * `endpoint.health.enabled` * `HttpSource.BasicLookupStrategy.send_head_requests` * `processor.purge_incompatible_from_source_cache` +2. Add the following methods from the sample delegate script: + * `jdbcsource_last_modified()` +3. If you are using a Java delegate, add the following method to your delegate + class: + * `getJDBCSourceLastModified()` ## 4.1.x → 5.0 diff --git a/delegates.rb.sample b/delegates.rb.sample index ad96bc633..d56cf1e9b 100644 --- a/delegates.rb.sample +++ b/delegates.rb.sample @@ -248,18 +248,33 @@ class CustomDelegate # should be used instead. # # @param options [Hash] Empty hash. - # @return [String] Identifier of the image corresponding to the given - # identifier in the database. + # @return [String, nil] Database identifier of the image corresponding to the + # identifier in the context, or nil if not found. # def jdbcsource_database_identifier(options = {}) end + ## + # Returns either the last-modified timestamp of an image in ISO 8601 format, + # or an SQL statement that can be used to retrieve it from a `TIMESTAMP`-type + # column in the database. In the latter case, the "SELECT" and "FROM" clauses + # should be in uppercase in order to be autodetected. + # + # Implementing this method is optional, but may be necessary for certain + # features (like `Last-Modified` response headers) to work. + # + # @param options [Hash] Empty hash. + # @return [String, nil] + # + def jdbcsource_last_modified(options = {}) + end + ## # Returns either the media (MIME) type of an image, or an SQL statement that - # can be used to retrieve it, if it is stored in the database. In the latter - # case, the "SELECT" and "FROM" clauses should be in uppercase in order to - # be autodetected. If nil is returned, the media type will be inferred some - # other way, such as by identifier extension or magic bytes. + # can be used to retrieve it from a `CHAR`-type column in the database. In + # the latter case, the "SELECT" and "FROM" clauses should be in uppercase in + # order to be autodetected. If nil is returned, the media type will be + # inferred some other way, such as by identifier extension or magic bytes. # # @param options [Hash] Empty hash. # @return [String, nil] diff --git a/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateMethod.java b/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateMethod.java index fd9adb6db..845ec7d67 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateMethod.java +++ b/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateMethod.java @@ -54,6 +54,13 @@ public enum DelegateMethod { */ JDBCSOURCE_MEDIA_TYPE("jdbcsource_media_type"), + /** + * Called by {@link DelegateProxy#getJdbcSourceLastModified()}. + * + * @since 6.0 + */ + JDBCSOURCE_LAST_MODIFIED("jdbcsource_last_modified"), + /** * Called by {@link DelegateProxy#getJdbcSourceLookupSQL()}. */ diff --git a/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateProxy.java b/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateProxy.java index 01730cb5d..1fee9ef7a 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateProxy.java +++ b/src/main/java/edu/illinois/library/cantaloupe/delegate/DelegateProxy.java @@ -87,6 +87,13 @@ Map getExtraIIIF3InformationResponseKeys() */ String getJdbcSourceDatabaseIdentifier() throws ScriptException; + /** + * @return Return value of {@link DelegateMethod#JDBCSOURCE_LAST_MODIFIED}. + * May be {@code null}. + * @since 6.0 + */ + String getJdbcSourceLastModified() throws ScriptException; + /** * @return Return value of {@link DelegateMethod#JDBCSOURCE_MEDIA_TYPE}. */ diff --git a/src/main/java/edu/illinois/library/cantaloupe/delegate/JRubyDelegateProxy.java b/src/main/java/edu/illinois/library/cantaloupe/delegate/JRubyDelegateProxy.java index d3a9fa949..cb3222ca6 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/delegate/JRubyDelegateProxy.java +++ b/src/main/java/edu/illinois/library/cantaloupe/delegate/JRubyDelegateProxy.java @@ -252,6 +252,16 @@ public String getJdbcSourceDatabaseIdentifier() throws ScriptException { return (String) result; } + /** + * @return Return value of {@link DelegateMethod#JDBCSOURCE_LAST_MODIFIED}. + * @since 6.0 + */ + @Override + public String getJdbcSourceLastModified() throws ScriptException { + Object result = invoke(DelegateMethod.JDBCSOURCE_LAST_MODIFIED); + return (String) result; + } + /** * @return Return value of {@link DelegateMethod#JDBCSOURCE_MEDIA_TYPE}. */ diff --git a/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaDelegate.java b/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaDelegate.java index 7a4b3f54e..7d825ae44 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaDelegate.java +++ b/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaDelegate.java @@ -204,13 +204,28 @@ public interface JavaDelegate { */ String getJDBCSourceDatabaseIdentifier(); + /** + *

      Returns either the last-modified timestamp of an image in ISO 8601 + * format, or an SQL statement that can be used to retrieve it from a + * {@code TIMESTAMP}-type column in the database. In the latter case, the + * {@code SELECT} and {@code FROM} clauses should be in uppercase in order + * to be autodetected.

      + * + *

      Implementing this method is optional, but may be necessary for + * certain features (like {@code Last-Modified} response headers) to + * work.

      + * + * @since 6.0 + */ + String getJDBCSourceLastModified(); + /** * Returns either the media (MIME) type of an image, or an SQL statement - * that can be used to retrieve it, if it is stored in the database. In the - * latter case, the {@code SELECT} and {@code FROM} clauses should be in - * uppercase in order to be autodetected. If {@code null} is returned, the - * media type will be inferred some other way, such as by identifier - * extension or magic bytes. + * that can be used to retrieve it from a {@code CHAR}-type column in the + * database. In the latter case, the {@code SELECT} and {@code FROM} + * clauses should be in uppercase in order to be autodetected. If {@code + * null} is returned, the media type will be inferred some other way, such + * as by identifier extension or magic bytes. */ String getJDBCSourceMediaType(); diff --git a/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaDelegateProxy.java b/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaDelegateProxy.java index 79140f038..f13c939ca 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaDelegateProxy.java +++ b/src/main/java/edu/illinois/library/cantaloupe/delegate/JavaDelegateProxy.java @@ -63,6 +63,14 @@ public String getJdbcSourceDatabaseIdentifier() { return delegate.getJDBCSourceDatabaseIdentifier(); } + /** + * @since 6.0 + */ + @Override + public String getJdbcSourceLastModified() { + return delegate.getJDBCSourceLastModified(); + } + @Override public String getJdbcSourceLookupSQL() { return delegate.getJDBCSourceLookupSQL(); diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandler.java b/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandler.java index 58af26d71..5641c7f8b 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandler.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandler.java @@ -15,6 +15,7 @@ import edu.illinois.library.cantaloupe.processor.ProcessorFactory; import edu.illinois.library.cantaloupe.processor.SourceFormatException; import edu.illinois.library.cantaloupe.delegate.DelegateProxy; +import edu.illinois.library.cantaloupe.source.StatResult; import edu.illinois.library.cantaloupe.status.HealthChecker; import edu.illinois.library.cantaloupe.source.Source; import edu.illinois.library.cantaloupe.source.SourceFactory; @@ -337,7 +338,7 @@ public void handle(OutputStream outputStream) throws Exception { final Optional sourceImage = cacheFacade.getSourceCacheFile(identifier); if (sourceImage.isEmpty() || isResolvingFirst()) { try { - source.checkAccess(); + StatResult result = source.stat(); } catch (NoSuchFileException e) { // this needs to be rethrown! if (config.getBoolean(Key.CACHE_SERVER_PURGE_MISSING, false)) { // If the image was not found, purge it from the cache. diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/InformationRequestHandler.java b/src/main/java/edu/illinois/library/cantaloupe/resource/InformationRequestHandler.java index 81591da82..68df5b008 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/InformationRequestHandler.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/InformationRequestHandler.java @@ -14,6 +14,7 @@ import edu.illinois.library.cantaloupe.delegate.DelegateProxy; import edu.illinois.library.cantaloupe.source.Source; import edu.illinois.library.cantaloupe.source.SourceFactory; +import edu.illinois.library.cantaloupe.source.StatResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -245,7 +246,7 @@ public Info handle() throws Exception { final Optional optSrcImage = cacheFacade.getSourceCacheFile(identifier); if (optSrcImage.isEmpty() || isResolvingFirst()) { try { - source.checkAccess(); + StatResult result = source.stat(); } catch (NoSuchFileException e) { // this needs to be rethrown! if (config.getBoolean(Key.CACHE_SERVER_PURGE_MISSING, false)) { // If the image was not found, purge it from the cache. diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/AzureStorageSource.java b/src/main/java/edu/illinois/library/cantaloupe/source/AzureStorageSource.java index cf3a57cbc..af45a0aa2 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/AzureStorageSource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/AzureStorageSource.java @@ -226,8 +226,11 @@ private static synchronized CloudBlobClient getClientInstance() { } @Override - public void checkAccess() throws IOException { - fetchBlob(); + public StatResult stat() throws IOException { + CloudBlockBlob blob = fetchBlob(); + StatResult result = new StatResult(); + result.setLastModified(blob.getProperties().getLastModified().toInstant()); + return result; } private CloudBlockBlob fetchBlob() throws IOException { @@ -275,6 +278,7 @@ String getBlobKey() throws IOException { if (objectKey == null) { final LookupStrategy strategy = LookupStrategy.from(Key.AZURESTORAGESOURCE_LOOKUP_STRATEGY); + //noinspection SwitchStatementWithTooFewBranches switch (strategy) { case DELEGATE_SCRIPT: try { diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/FilesystemSource.java b/src/main/java/edu/illinois/library/cantaloupe/source/FilesystemSource.java index d2b4a4bb2..ab6cafd6a 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/FilesystemSource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/FilesystemSource.java @@ -118,7 +118,7 @@ public T next() { private Path path; @Override - public void checkAccess() throws IOException { + public StatResult stat() throws IOException { final Path file = getFile(); if (!Files.exists(file)) { throw new NoSuchFileException("Failed to resolve " + @@ -126,6 +126,9 @@ public void checkAccess() throws IOException { } else if (!Files.isReadable(file)) { throw new AccessDeniedException("File is not readable: " + file); } + StatResult result = new StatResult(); + result.setLastModified(Files.getLastModifiedTime(file).toInstant()); + return result; } /** @@ -139,6 +142,7 @@ public Path getFile() throws IOException { if (path == null) { final LookupStrategy strategy = LookupStrategy.from(Key.FILESYSTEMSOURCE_LOOKUP_STRATEGY); + //noinspection SwitchStatementWithTooFewBranches switch (strategy) { case DELEGATE_SCRIPT: try { @@ -169,7 +173,7 @@ private Path getPathWithBasicStrategy() { final String suffix = config.getString(Key.FILESYSTEMSOURCE_PATH_SUFFIX, ""); final Identifier sanitizedId = sanitizedIdentifier(); - return Paths.get(prefix + sanitizedId.toString() + suffix); + return Paths.get(prefix + sanitizedId + suffix); } /** diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java b/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java index 8edcc20ed..5208a6314 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java @@ -30,9 +30,14 @@ import java.security.SecureRandom; import java.security.cert.X509Certificate; import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.temporal.TemporalAccessor; import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; import java.util.concurrent.TimeUnit; @@ -170,6 +175,18 @@ Format detectFormat() throws IOException { return format; } + Instant lastModified() { + String str = headers.get("Last-Modified"); + if (str != null) { + TemporalAccessor ta = DateTimeFormatter.RFC_1123_DATE_TIME + .withLocale(Locale.UK) + .withZone(ZoneId.systemDefault()) + .parse(str); + return Instant.from(ta); + } + return null; + } + } /** @@ -518,9 +535,9 @@ static String toString(Headers headers) { } @Override - public void checkAccess() throws IOException { + public StatResult stat() throws IOException { ResourceInfo info = getResourceInfo(); - final int status = info.status; + final int status = info.status; if (status >= 400) { final String statusLine = "HTTP " + status; if (status == 404 || status == 410) { // not found or gone @@ -531,6 +548,9 @@ public void checkAccess() throws IOException { throw new IOException(statusLine); } } + StatResult result = new StatResult(); + result.setLastModified(info.lastModified()); + return result; } @Override diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/JdbcSource.java b/src/main/java/edu/illinois/library/cantaloupe/source/JdbcSource.java index faca97960..d937a391b 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/JdbcSource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/JdbcSource.java @@ -21,6 +21,8 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.Instant; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; @@ -28,7 +30,7 @@ /** *

      Maps an identifier to a binary/BLOB field in a relational database.

      * - *

      A custom schema is not required; any schema will work. However, several + *

      A custom schema is not required; most schemas will work. However, several * delegate methods must be implemented in order to obtain the information * needed to run the SQL queries.

      * @@ -211,23 +213,11 @@ static synchronized Connection getConnection() throws SQLException { } @Override - public void checkAccess() throws IOException { - try (Connection connection = getConnection()) { - final String sql = getLookupSQL(); - - // Check that the image exists. - try (PreparedStatement statement = connection.prepareStatement(sql)) { - statement.setString(1, getDatabaseIdentifier()); - LOGGER.debug(sql); - try (ResultSet result = statement.executeQuery()) { - if (!result.next()) { - throw new NoSuchFileException(sql); - } - } - } - } catch (ScriptException | SQLException e) { - throw new IOException(e); - } + public StatResult stat() throws IOException { + Instant lastModified = getLastModified(); + StatResult result = new StatResult(); + result.setLastModified(lastModified); + return result; } /** @@ -243,6 +233,41 @@ public FormatIterator getFormatIterator() { return formatIterator; } + Instant getLastModified() throws IOException { + try { + String methodResult = getDelegateProxy().getJdbcSourceLastModified(); + if (methodResult != null) { + // the delegate method result may be an ISO 8601 string, or an + // SQL statement to look it up. + if (methodResult.toUpperCase().startsWith("SELECT")) { + // It's called readability, IntelliJ! + //noinspection UnnecessaryLocalVariable + final String sql = methodResult; + LOGGER.debug(sql); + try (Connection connection = getConnection(); + PreparedStatement statement = connection.prepareStatement(sql)) { + statement.setString(1, getDatabaseIdentifier()); + try (ResultSet resultSet = statement.executeQuery()) { + if (resultSet.next()) { + Timestamp value = resultSet.getTimestamp(1); + if (value != null) { + return value.toInstant(); + } + } else { + throw new NoSuchFileException(sql); + } + } + } + } else { + return Instant.parse(methodResult); + } + } + } catch (SQLException | ScriptException e) { + throw new IOException(e); + } + return null; + } + /** * @return Result of the {@link DelegateMethod#JDBCSOURCE_LOOKUP_SQL} * method. diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/S3Source.java b/src/main/java/edu/illinois/library/cantaloupe/source/S3Source.java index ac60a4395..93db8621f 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/S3Source.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/S3Source.java @@ -28,6 +28,7 @@ import java.net.URISyntaxException; import java.nio.file.AccessDeniedException; import java.nio.file.NoSuchFileException; +import java.time.Instant; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -85,6 +86,7 @@ final class S3Source extends AbstractSource implements Source { private static class S3ObjectAttributes { String contentType; + Instant lastModified; long length; } @@ -298,8 +300,11 @@ static InputStream newObjectInputStream(S3ObjectInfo info, } @Override - public void checkAccess() throws IOException { - getObjectAttributes(); + public StatResult stat() throws IOException { + S3ObjectAttributes attrs = getObjectAttributes(); + StatResult result = new StatResult(); + result.setLastModified(attrs.lastModified); + return result; } @Override @@ -319,8 +324,9 @@ private S3ObjectAttributes getObjectAttributes() throws IOException { .bucket(bucket) .key(key) .build()); - objectAttributes = new S3ObjectAttributes(); - objectAttributes.length = response.contentLength(); + objectAttributes = new S3ObjectAttributes(); + objectAttributes.length = response.contentLength(); + objectAttributes.lastModified = response.lastModified(); } catch (NoSuchBucketException | NoSuchKeyException e) { throw new NoSuchFileException(info.toString()); } catch (S3Exception e) { @@ -346,6 +352,7 @@ private S3ObjectAttributes getObjectAttributes() throws IOException { */ S3ObjectInfo getObjectInfo() throws IOException { if (objectInfo == null) { + //noinspection SwitchStatementWithTooFewBranches switch (LookupStrategy.from(Key.S3SOURCE_LOOKUP_STRATEGY)) { case DELEGATE_SCRIPT: try { diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/Source.java b/src/main/java/edu/illinois/library/cantaloupe/source/Source.java index bc21d30f9..f3901a31e 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/Source.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/Source.java @@ -20,7 +20,7 @@ *
        *
      1. {@link #setIdentifier(Identifier)} and * {@link #setDelegateProxy(DelegateProxy)} (in either order)
        2. - *
      2. {@link #checkAccess()}
        3. + *
      3. {@link #stat()}
        4. *
      4. Any other methods
        5. *
      5. {@link #shutdown()}
        6. *
      @@ -39,17 +39,19 @@ public interface Source { void setIdentifier(Identifier identifier); /** - *

      Checks the accessibility of the source image.

      + *

      Checks the accessibility of the source image and returns some limited + * metadata.

      * *

      Will be called only once.

      * + * @return Instance with as many of its properties set as possible. * @throws NoSuchFileException if an image corresponding to the set * identifier does not exist. * @throws AccessDeniedException if an image corresponding to the set * identifier is not readable. * @throws IOException if there is some other issue accessing the image. */ - void checkAccess() throws IOException; + StatResult stat() throws IOException; /** * N.B.: This default implementation throws an {@link @@ -104,10 +106,11 @@ default Path getFile() throws IOException { default void shutdown() {} /** - * N.B. 1: This method's return value affects the behavior of {@link - * #getFile()}. See the documentation of that method for more information. + *

      N.B. 1: This method's return value affects the behavior of {@link + * #getFile()}. See the documentation of that method for more + * information.

      * - * N.B. 2: The default implementation returns {@code false}. + *

      N.B. 2: The default implementation returns {@code false}.

      * * @return Whether the source image can be accessed via a {@link * java.nio.file.Path}. diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/StatResult.java b/src/main/java/edu/illinois/library/cantaloupe/source/StatResult.java new file mode 100644 index 000000000..f9f5710da --- /dev/null +++ b/src/main/java/edu/illinois/library/cantaloupe/source/StatResult.java @@ -0,0 +1,20 @@ +package edu.illinois.library.cantaloupe.source; + +import java.time.Instant; + +/** + * Holds some limited metadata about a source image. + */ +public final class StatResult { + + private Instant lastModified; + + public Instant getLastModified() { + return lastModified; + } + + void setLastModified(Instant lastModified) { + this.lastModified = lastModified; + } + +} diff --git a/src/test/java/edu/illinois/library/cantaloupe/delegate/JRubyDelegateProxyTest.java b/src/test/java/edu/illinois/library/cantaloupe/delegate/JRubyDelegateProxyTest.java index dcfa39d1b..5252db1de 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/delegate/JRubyDelegateProxyTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/delegate/JRubyDelegateProxyTest.java @@ -237,6 +237,14 @@ void testGetJdbcSourceDatabaseIdentifier() throws Exception { assertEquals("cats", result); } + /* getJdbcSourceLastModified() */ + + @Test + void testGetJdbcSourceLastModified() throws Exception { + String result = instance.getJdbcSourceLastModified(); + assertEquals("SELECT last_modified FROM items WHERE filename = ?", result); + } + /* getJdbcSourceMediaType() */ @Test diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/ImageAPIResourceTester.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/ImageAPIResourceTester.java index abdc24667..5162b9c96 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/ImageAPIResourceTester.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/ImageAPIResourceTester.java @@ -16,6 +16,7 @@ import edu.illinois.library.cantaloupe.source.AccessDeniedSource; import edu.illinois.library.cantaloupe.source.PathStreamFactory; import edu.illinois.library.cantaloupe.source.Source; +import edu.illinois.library.cantaloupe.source.StatResult; import edu.illinois.library.cantaloupe.source.StreamFactory; import edu.illinois.library.cantaloupe.delegate.DelegateProxy; import edu.illinois.library.cantaloupe.test.TestUtil; @@ -359,7 +360,7 @@ public void testRecoveryFromIncorrectSourceFormat(URI uri) throws Exception { public static class NotCheckingAccessSource implements Source { @Override - public void checkAccess() throws IOException { + public StatResult stat() throws IOException { throw new IOException("checkAccess called!"); } @@ -434,7 +435,9 @@ public void testSourceCheckAccessNotCalledWithSourceCacheHit(Identifier identifi public static class NotReadingSourceFormatSource implements Source { @Override - public void checkAccess() {} + public StatResult stat() { + return null; + } @Override public Iterator getFormatIterator() { diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/AbstractSourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/AbstractSourceTest.java index 1096dacf0..c9084cc46 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/AbstractSourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/AbstractSourceTest.java @@ -25,89 +25,105 @@ public void setUp() throws Exception { useBasicLookupStrategy(); } - /* checkAccess() */ + /* getFormatIterator() */ @Test - void testCheckAccessUsingBasicLookupStrategyWithPresentReadableImage() - throws Exception { + void testGetFormatIteratorConsecutiveInvocationsReturnSameInstance() { + Source instance = newInstance(); + var it = instance.getFormatIterator(); + assertSame(it, instance.getFormatIterator()); + } + + /* newStreamFactory() */ + + @Test + void testNewStreamFactoryInvokedMultipleTimes() throws Exception { + Source instance = newInstance(); try { initializeEndpoint(); - - newInstance().checkAccess(); + instance.newStreamFactory(); + instance.newStreamFactory(); + instance.newStreamFactory(); } finally { destroyEndpoint(); } } @Test - void testCheckAccessUsingBasicLookupStrategyWithMissingImage() + void testNewStreamFactoryReturnedInstanceIsReusable() throws Exception { + Source instance = newInstance(); try { initializeEndpoint(); + StreamFactory source = instance.newStreamFactory(); - Source instance = newInstance(); - instance.setIdentifier(new Identifier("bogus")); - assertThrows(NoSuchFileException.class, instance::checkAccess); + try (InputStream is = source.newInputStream(); + OutputStream os = OutputStream.nullOutputStream()) { + is.transferTo(os); + } + + try (InputStream is = source.newInputStream(); + OutputStream os = OutputStream.nullOutputStream()) { + is.transferTo(os); + } } finally { destroyEndpoint(); } } + /* stat() */ + @Test - void testCheckAccessInvokedMultipleTimes() throws Exception { + void testStatUsingBasicLookupStrategyWithPresentReadableImage() + throws Exception { try { initializeEndpoint(); - Source instance = newInstance(); - instance.checkAccess(); - instance.checkAccess(); - instance.checkAccess(); + newInstance().stat(); } finally { destroyEndpoint(); } } - /* getFormatIterator() */ - @Test - void testGetFormatIteratorConsecutiveInvocationsReturnSameInstance() { - Source instance = newInstance(); - var it = instance.getFormatIterator(); - assertSame(it, instance.getFormatIterator()); - } + void testStatUsingBasicLookupStrategyWithMissingImage() + throws Exception { + try { + initializeEndpoint(); - /* newStreamFactory() */ + Source instance = newInstance(); + instance.setIdentifier(new Identifier("bogus")); + assertThrows(NoSuchFileException.class, instance::stat); + } finally { + destroyEndpoint(); + } + } @Test - void testNewStreamFactoryInvokedMultipleTimes() throws Exception { - Source instance = newInstance(); + void testStatReturnsCorrectInstance() throws Exception { try { initializeEndpoint(); - instance.newStreamFactory(); - instance.newStreamFactory(); - instance.newStreamFactory(); + + StatResult result = newInstance().stat(); + assertNotNull(result.getLastModified()); } finally { destroyEndpoint(); } } + /** + * Tests that {@link Source#stat()} can be invoked multiple times without + * throwing an exception. + */ @Test - void testNewStreamFactoryReturnedInstanceIsReusable() - throws Exception { - Source instance = newInstance(); + void testStatInvokedMultipleTimes() throws Exception { try { initializeEndpoint(); - StreamFactory source = instance.newStreamFactory(); - try (InputStream is = source.newInputStream(); - OutputStream os = OutputStream.nullOutputStream()) { - is.transferTo(os); - } - - try (InputStream is = source.newInputStream(); - OutputStream os = OutputStream.nullOutputStream()) { - is.transferTo(os); - } + Source instance = newInstance(); + instance.stat(); + instance.stat(); + instance.stat(); } finally { destroyEndpoint(); } diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/AccessDeniedSource.java b/src/test/java/edu/illinois/library/cantaloupe/source/AccessDeniedSource.java index 02c017235..1acb57949 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/AccessDeniedSource.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/AccessDeniedSource.java @@ -9,7 +9,7 @@ public class AccessDeniedSource extends AbstractSource implements Source { @Override - public void checkAccess() throws IOException { + public StatResult stat() throws IOException { throw new AccessDeniedException(""); } diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/AzureStorageSourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/AzureStorageSourceTest.java index fdff3bbc7..7f51989e2 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/AzureStorageSourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/AzureStorageSourceTest.java @@ -169,7 +169,7 @@ void testCheckAccessUsingBasicLookupStrategyWithPresentUnreadableImage() { void testCheckAccessUsingScriptLookupStrategyWithPresentReadableImage() throws Exception { useScriptLookupStrategy(); - instance.checkAccess(); + instance.stat(); } @Test @@ -188,14 +188,14 @@ void testCheckAccessUsingScriptLookupStrategyWithMissingImage() { instance.setDelegateProxy(delegateProxy); instance.setIdentifier(identifier); - assertThrows(NoSuchFileException.class, instance::checkAccess); + assertThrows(NoSuchFileException.class, instance::stat); } @Test void testCheckAccessWithSAS() throws Exception { instance.setIdentifier(new Identifier(getSASURI())); clearConfig(); - instance.checkAccess(); + instance.stat(); } /* getFormatIterator() */ diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/FilesystemSourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/FilesystemSourceTest.java index d3401fd7a..5db555695 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/FilesystemSourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/FilesystemSourceTest.java @@ -77,7 +77,7 @@ void testCheckAccessUsingBasicLookupStrategyWithPresentUnreadableFile() Path path = instance.getFile(); try { assumeTrue(path.toFile().setReadable(false)); - assertThrows(AccessDeniedException.class, instance::checkAccess); + assertThrows(AccessDeniedException.class, instance::stat); } finally { path.toFile().setReadable(true); } @@ -95,7 +95,7 @@ void testCheckAccessUsingScriptLookupStrategyWithPresentReadableFile() instance.setDelegateProxy(proxy); instance.setIdentifier(identifier); - instance.checkAccess(); + instance.stat(); } @Test @@ -114,7 +114,7 @@ void testCheckAccessUsingScriptLookupStrategyWithPresentUnreadableFile() try { assumeTrue(path.toFile().setReadable(false)); Files.setPosixFilePermissions(path, Collections.emptySet()); - assertThrows(AccessDeniedException.class, instance::checkAccess); + assertThrows(AccessDeniedException.class, instance::stat); } finally { path.toFile().setReadable(true); } @@ -129,7 +129,7 @@ void testCheckAccessUsingScriptLookupStrategyWithMissingFile() { proxy.getRequestContext().setIdentifier(identifier); instance.setDelegateProxy(proxy); instance.setIdentifier(identifier); - assertThrows(NoSuchFileException.class, instance::checkAccess); + assertThrows(NoSuchFileException.class, instance::stat); } /* getFormatIterator() */ diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java index cd1ad104d..ccbfda23b 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java @@ -159,7 +159,7 @@ private void doTestCheckAccessWithPresentReadableImage(Identifier identifier) instance.setDelegateProxy(proxy); instance.setIdentifier(identifier); - instance.checkAccess(); + instance.stat(); } private void doTestCheckAccessWithPresentUnreadableImage(Identifier identifier) @@ -181,7 +181,7 @@ public void handle(String target, instance.setDelegateProxy(proxy); instance.setIdentifier(identifier); instance.setIdentifier(identifier); - assertThrows(AccessDeniedException.class, instance::checkAccess); + assertThrows(AccessDeniedException.class, instance::stat); } private void doTestCheckAccessWithMissingImage(Identifier identifier) @@ -194,7 +194,7 @@ private void doTestCheckAccessWithMissingImage(Identifier identifier) instance.setDelegateProxy(proxy); instance.setIdentifier(identifier); - instance.checkAccess(); + instance.stat(); fail("Expected exception"); } catch (NoSuchFileException e) { // pass @@ -216,7 +216,7 @@ void testCheckAccessUsingScriptLookupStrategyWithValidAuthentication() instance.setDelegateProxy(proxy); instance.setIdentifier(identifier); - instance.checkAccess(); + instance.stat(); } @Test @@ -234,7 +234,7 @@ void testCheckAccessUsingScriptLookupStrategyWithInvalidAuthentication() instance.setDelegateProxy(proxy); instance.setIdentifier(identifier); - assertThrows(AccessDeniedException.class, instance::checkAccess); + assertThrows(AccessDeniedException.class, instance::stat); } @Test @@ -253,7 +253,7 @@ public void handle(String target, try { instance.setIdentifier(PRESENT_READABLE_IDENTIFIER); - instance.checkAccess(); + instance.stat(); fail("Expected exception"); } catch (AccessDeniedException e) { assertTrue(e.getMessage().contains("403")); @@ -276,7 +276,7 @@ public void handle(String target, try { instance.setIdentifier(PRESENT_READABLE_IDENTIFIER); - instance.checkAccess(); + instance.stat(); fail("Expected exception"); } catch (IOException e) { assertTrue(e.getMessage().contains("500")); @@ -305,7 +305,7 @@ public void handle(String target, }); server.start(); - instance.checkAccess(); + instance.stat(); } @Test @@ -331,7 +331,7 @@ public void handle(String target, instance.setDelegateProxy(proxy); instance.setIdentifier(identifier); - instance.checkAccess(); + instance.stat(); } @Test @@ -346,7 +346,7 @@ void testCheckAccessWithMalformedURI() throws Exception { PRESENT_READABLE_IDENTIFIER); instance.setIdentifier(identifier); - assertThrows(IOException.class, () -> instance.checkAccess()); + assertThrows(IOException.class, () -> instance.stat()); } /* getFormatIterator() */ @@ -589,7 +589,7 @@ void testNoUnnecessaryRequestsWithHEADRequestsEnabled() throws Exception { server.setHandler(handler); server.start(); - instance.checkAccess(); + instance.stat(); instance.getFormatIterator().next(); StreamFactory source = instance.newStreamFactory(); @@ -614,7 +614,7 @@ void testNoUnnecessaryRequestsWithHEADRequestsDisabled() throws Exception { server.setHandler(handler); server.start(); - instance.checkAccess(); + instance.stat(); instance.getFormatIterator().next(); StreamFactory source = instance.newStreamFactory(); diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/JdbcSourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/JdbcSourceTest.java index 2e14cbcf9..78a8eddad 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/JdbcSourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/JdbcSourceTest.java @@ -16,7 +16,9 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.sql.Timestamp; import java.sql.Types; +import java.time.Instant; import java.util.NoSuchElementException; import static org.junit.jupiter.api.Assertions.*; @@ -73,6 +75,7 @@ void initializeEndpoint() throws Exception { String sql = "CREATE TABLE IF NOT EXISTS items (" + "filename VARCHAR(255)," + "media_type VARCHAR(255)," + + "last_modified TIMESTAMP," + "image BLOB);"; try (PreparedStatement statement = conn.prepareStatement(sql)) { statement.execute(); @@ -85,19 +88,21 @@ void initializeEndpoint() throws Exception { IMAGE_WITH_EXTENSION_WITHOUT_MEDIA_TYPE, IMAGE_WITH_INCORRECT_EXTENSION_WITHOUT_MEDIA_TYPE, IMAGE_WITHOUT_EXTENSION_OR_MEDIA_TYPE}) { - sql = "INSERT INTO items (filename, media_type, image) VALUES (?, ?, ?);"; + sql = "INSERT INTO items (filename, last_modified, media_type, image) " + + "VALUES (?, ?, ?, ?);"; try (PreparedStatement statement = conn.prepareStatement(sql)) { statement.setString(1, filename); + statement.setTimestamp(2, Timestamp.from(Instant.now())); if (IMAGE_WITHOUT_EXTENSION_OR_MEDIA_TYPE.equals(filename) || IMAGE_WITH_EXTENSION_WITHOUT_MEDIA_TYPE.equals(filename) || IMAGE_WITH_INCORRECT_EXTENSION_WITHOUT_MEDIA_TYPE.equals(filename)) { - statement.setNull(2, Types.VARCHAR); + statement.setNull(3, Types.VARCHAR); } else { - statement.setString(2, "image/jpeg"); + statement.setString(3, "image/jpeg"); } try (InputStream is = Files.newInputStream(TestUtil.getImage("jpg"))) { - statement.setBinaryStream(3, is); + statement.setBinaryStream(4, is); } statement.executeUpdate(); } @@ -126,20 +131,6 @@ void useScriptLookupStrategy() { // This source is always using ScriptLookupStrategy. } - /* checkAccess() */ - - @Override - @Test - void testCheckAccessUsingBasicLookupStrategyWithMissingImage() { - Identifier identifier = new Identifier("bogus"); - DelegateProxy proxy = TestUtil.newDelegateProxy(); - proxy.getRequestContext().setIdentifier(identifier); - instance.setDelegateProxy(proxy); - instance.setIdentifier(identifier); - - assertThrows(NoSuchFileException.class, instance::checkAccess); - } - /* getFormatIterator() */ @Test @@ -227,4 +218,18 @@ void testNewStreamFactoryWithPresentImage() throws Exception { assertNotNull(instance.newStreamFactory()); } + /* stat() */ + + @Override + @Test + void testStatUsingBasicLookupStrategyWithMissingImage() { + Identifier identifier = new Identifier("bogus"); + DelegateProxy proxy = TestUtil.newDelegateProxy(); + proxy.getRequestContext().setIdentifier(identifier); + instance.setDelegateProxy(proxy); + instance.setIdentifier(identifier); + + assertThrows(NoSuchFileException.class, instance::stat); + } + } diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/MockFileSource.java b/src/test/java/edu/illinois/library/cantaloupe/source/MockFileSource.java index cdb989640..b1d556497 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/MockFileSource.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/MockFileSource.java @@ -11,7 +11,8 @@ public class MockFileSource extends AbstractSource implements Source { @Override - public void checkAccess() throws IOException { + public StatResult stat() throws IOException { + return null; } @Override diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/MockStreamSource.java b/src/test/java/edu/illinois/library/cantaloupe/source/MockStreamSource.java index 402e8c0a5..5ecbfa6de 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/MockStreamSource.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/MockStreamSource.java @@ -9,7 +9,8 @@ public class MockStreamSource extends AbstractSource implements Source { @Override - public void checkAccess() { + public StatResult stat() { + return null; } @Override diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/S3SourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/S3SourceTest.java index 56d42aa35..5a91fad41 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/S3SourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/S3SourceTest.java @@ -241,7 +241,7 @@ void checkAccessUsingBasicLookupStrategyWithPresentUnreadableImage() { void checkAccessUsingScriptLookupStrategyWithPresentReadableImage() throws Exception { useScriptLookupStrategy(); - instance.checkAccess(); + instance.stat(); } @Test @@ -259,7 +259,7 @@ void checkAccessUsingScriptLookupStrategyWithMissingImage() { instance.setDelegateProxy(proxy); instance.setIdentifier(identifier); - assertThrows(NoSuchFileException.class, () -> instance.checkAccess()); + assertThrows(NoSuchFileException.class, () -> instance.stat()); } @Test @@ -274,7 +274,7 @@ void checkAccessUsingScriptLookupStrategyReturningHash() throws Exception { proxy.getRequestContext().setIdentifier(identifier); instance.setDelegateProxy(proxy); - instance.checkAccess(); + instance.stat(); } @Test @@ -287,7 +287,7 @@ void checkAccessUsingScriptLookupStrategyWithMissingKeyInReturnedHash() { instance.setDelegateProxy(proxy); instance.setIdentifier(identifier); - assertThrows(IllegalArgumentException.class, () -> instance.checkAccess()); + assertThrows(IllegalArgumentException.class, () -> instance.stat()); } /* getFormatIterator() */ diff --git a/src/test/resources/delegates.rb b/src/test/resources/delegates.rb index e7f0ea4e9..3ad145081 100644 --- a/src/test/resources/delegates.rb +++ b/src/test/resources/delegates.rb @@ -220,6 +220,10 @@ def jdbcsource_database_identifier(options = {}) context['identifier'] end + def jdbcsource_last_modified(options = {}) + 'SELECT last_modified FROM items WHERE filename = ?' + end + def jdbcsource_media_type(options = {}) 'SELECT media_type FROM items WHERE filename = ?' end From 9cf7cd4bac3feb687ccb71e2bc2568176c690798 Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Mon, 21 Mar 2022 14:34:22 -0500 Subject: [PATCH 032/106] Add Last-Modified header to image & information responses (#571) --- CHANGES.md | 5 +-- .../resource/ImageRequestHandler.java | 11 ++++++ .../resource/InformationRequestHandler.java | 15 ++++++- .../resource/iiif/IIIFResource.java | 13 +++++++ .../resource/iiif/v1/ImageResource.java | 8 ++++ .../resource/iiif/v1/InformationResource.java | 19 ++++----- .../resource/iiif/v2/ImageResource.java | 8 ++++ .../resource/iiif/v2/InformationResource.java | 19 ++++----- .../resource/iiif/v3/ImageResource.java | 8 ++++ .../resource/iiif/v3/InformationResource.java | 19 ++++----- .../resource/ImageRequestHandlerTest.java | 39 +++++++++++++++++++ .../InformationRequestHandlerTest.java | 29 ++++++++++++++ .../resource/iiif/ImageAPIResourceTester.java | 14 ++++++- .../resource/iiif/v1/ImageResourceTest.java | 4 +- .../iiif/v1/InformationResourceTest.java | 6 +-- .../resource/iiif/v2/ImageResourceTest.java | 4 +- .../iiif/v2/InformationResourceTest.java | 6 +-- .../resource/iiif/v3/ImageResourceTest.java | 4 +- .../iiif/v3/InformationResourceTest.java | 6 +-- 19 files changed, 192 insertions(+), 45 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 12e85f2be..c8ddd1178 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,9 +2,8 @@ # 6.0 -* Responses for infos serialized in this version or later by any cache, or in - earlier versions by AzureStorageCache, FilesystemCache, or S3Cache, include a - `Last-Modified` header. +* Image and information responses include a `Last-Modified` header when + possible. * The health endpoint is enabled via `endpoint.health.enabled` rather than `endpoint.api.enabled`. * HttpSource can be configured to send a ranged GET request instead of a HEAD diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandler.java b/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandler.java index 5641c7f8b..57a7b2cf1 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandler.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandler.java @@ -184,6 +184,13 @@ public interface Callback { */ boolean authorize() throws Exception; + /** + * Called immediately after the source image has first been accessed. + * + * @param result Information about the source image. + */ + void sourceAccessed(StatResult result); + /** * Called when image information is available; always before {@link * #willProcessImage(Processor, Info)} and {@link @@ -233,6 +240,9 @@ public boolean authorize() { return true; } @Override + public void sourceAccessed(StatResult result) { + } + @Override public void willStreamImageFromDerivativeCache() { } @Override @@ -339,6 +349,7 @@ public void handle(OutputStream outputStream) throws Exception { if (sourceImage.isEmpty() || isResolvingFirst()) { try { StatResult result = source.stat(); + callback.sourceAccessed(result); } catch (NoSuchFileException e) { // this needs to be rethrown! if (config.getBoolean(Key.CACHE_SERVER_PURGE_MISSING, false)) { // If the image was not found, purge it from the cache. diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/InformationRequestHandler.java b/src/main/java/edu/illinois/library/cantaloupe/resource/InformationRequestHandler.java index 68df5b008..b79552d6d 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/InformationRequestHandler.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/InformationRequestHandler.java @@ -51,7 +51,7 @@ public class InformationRequestHandler extends AbstractRequestHandler */ public static final class Builder { - private InformationRequestHandler handler; + private final InformationRequestHandler handler; private Builder(InformationRequestHandler handler) { this.handler = handler; @@ -148,7 +148,14 @@ public interface Callback { boolean authorize() throws Exception; /** - * Called when the list of available output formats have been obtained + * Called immediately after a source image has been accessed. + * + * @param result Information about the source image. + */ + void sourceAccessed(StatResult result); + + /** + * Called when a list of available output formats has been obtained * from a {@link Processor}. */ void knowAvailableOutputFormats(Set availableOutputFormats); @@ -165,6 +172,9 @@ public boolean authorize() { return true; } @Override + public void sourceAccessed(StatResult sourceAvailable) { + } + @Override public void knowAvailableOutputFormats(Set availableOutputFormats) { } }; @@ -247,6 +257,7 @@ public Info handle() throws Exception { if (optSrcImage.isEmpty() || isResolvingFirst()) { try { StatResult result = source.stat(); + callback.sourceAccessed(result); } catch (NoSuchFileException e) { // this needs to be rethrown! if (config.getBoolean(Key.CACHE_SERVER_PURGE_MISSING, false)) { // If the image was not found, purge it from the cache. diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/IIIFResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/IIIFResource.java index 0bc631794..24e1b0b19 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/IIIFResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/IIIFResource.java @@ -9,8 +9,21 @@ import edu.illinois.library.cantaloupe.operation.ScaleByPixels; import edu.illinois.library.cantaloupe.resource.PublicResource; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Locale; + public abstract class IIIFResource extends PublicResource { + protected void setLastModifiedHeader(Instant lastModified) { + getResponse().setHeader("Last-Modified", + DateTimeFormatter.RFC_1123_DATE_TIME + .withLocale(Locale.UK) + .withZone(ZoneId.systemDefault()) + .format(lastModified)); + } + /** * When the size expressed in the endpoint URI is {@code max}, and the * resulting image dimensions are larger than {@link Key#MAX_PIXELS}, the diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageResource.java index 0b6ab0cbc..1bcb9fc55 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageResource.java @@ -12,6 +12,7 @@ import edu.illinois.library.cantaloupe.operation.Scale; import edu.illinois.library.cantaloupe.processor.Processor; import edu.illinois.library.cantaloupe.resource.ImageRequestHandler; +import edu.illinois.library.cantaloupe.source.StatResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,6 +77,13 @@ public boolean authorize() throws Exception { return ImageResource.this.authorize(); } + @Override + public void sourceAccessed(StatResult result) { + if (result.getLastModified() != null) { + setLastModifiedHeader(result.getLastModified()); + } + } + @Override public void infoAvailable(Info info) { } diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResource.java index 3e6903cf7..143daad95 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResource.java @@ -1,11 +1,7 @@ package edu.illinois.library.cantaloupe.resource.iiif.v1; -import java.time.Instant; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Set; import edu.illinois.library.cantaloupe.http.Method; @@ -15,6 +11,7 @@ import edu.illinois.library.cantaloupe.resource.ResourceException; import edu.illinois.library.cantaloupe.resource.Route; import edu.illinois.library.cantaloupe.resource.InformationRequestHandler; +import edu.illinois.library.cantaloupe.source.StatResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,6 +63,14 @@ public boolean authorize() throws Exception { } return false; } + + @Override + public void sourceAccessed(StatResult result) { + if (result.getLastModified() != null) { + setLastModifiedHeader(result.getLastModified()); + } + } + @Override public void knowAvailableOutputFormats(Set formats) { availableOutputFormats.addAll(formats); @@ -103,11 +108,7 @@ private void addHeaders(Info info, Information iiifInfo) { String.format("<%s>;rel=\"profile\";", iiifInfo.profile)); // Last-Modified if (info.getSerializationTimestamp() != null) { - getResponse().setHeader("Last-Modified", - DateTimeFormatter.RFC_1123_DATE_TIME - .withLocale(Locale.UK) - .withZone(ZoneId.systemDefault()) - .format(info.getSerializationTimestamp())); + setLastModifiedHeader(info.getSerializationTimestamp()); } } diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageResource.java index 12bbc20fc..5673c699a 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageResource.java @@ -15,6 +15,7 @@ import edu.illinois.library.cantaloupe.resource.Route; import edu.illinois.library.cantaloupe.resource.ImageRequestHandler; import edu.illinois.library.cantaloupe.resource.iiif.SizeRestrictedException; +import edu.illinois.library.cantaloupe.source.StatResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -84,6 +85,13 @@ public boolean authorize() throws Exception { return ImageResource.this.authorize(); } + @Override + public void sourceAccessed(StatResult result) { + if (result.getLastModified() != null) { + setLastModifiedHeader(result.getLastModified()); + } + } + @Override public void infoAvailable(Info info) { if (Size.ScaleMode.MAX.equals(params.getSize().getScaleMode())) { diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResource.java index daea7fa32..4fc187dbd 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResource.java @@ -1,11 +1,7 @@ package edu.illinois.library.cantaloupe.resource.iiif.v2; -import java.time.Instant; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Set; import edu.illinois.library.cantaloupe.http.Method; @@ -16,6 +12,7 @@ import edu.illinois.library.cantaloupe.resource.ResourceException; import edu.illinois.library.cantaloupe.resource.Route; import edu.illinois.library.cantaloupe.resource.InformationRequestHandler; +import edu.illinois.library.cantaloupe.source.StatResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,6 +68,14 @@ public boolean authorize() throws Exception { } return false; } + + @Override + public void sourceAccessed(StatResult result) { + if (result.getLastModified() != null) { + setLastModifiedHeader(result.getLastModified()); + } + } + @Override public void knowAvailableOutputFormats(Set formats) { availableOutputFormats.addAll(formats); @@ -97,11 +102,7 @@ private void addHeaders(Info info) { getResponse().setHeader("Content-Type", getNegotiatedMediaType()); // Last-Modified if (info.getSerializationTimestamp() != null) { - getResponse().setHeader("Last-Modified", - DateTimeFormatter.RFC_1123_DATE_TIME - .withLocale(Locale.UK) - .withZone(ZoneId.systemDefault()) - .format(info.getSerializationTimestamp())); + setLastModifiedHeader(info.getSerializationTimestamp()); } } diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageResource.java index 40ebf3fa7..432777472 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageResource.java @@ -17,6 +17,7 @@ import edu.illinois.library.cantaloupe.resource.ScaleRestrictedException; import edu.illinois.library.cantaloupe.resource.ImageRequestHandler; import edu.illinois.library.cantaloupe.resource.iiif.SizeRestrictedException; +import edu.illinois.library.cantaloupe.source.StatResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -87,6 +88,13 @@ public boolean authorize() throws Exception { return ImageResource.this.authorize(); } + @Override + public void sourceAccessed(StatResult result) { + if (result.getLastModified() != null) { + setLastModifiedHeader(result.getLastModified()); + } + } + @Override public void infoAvailable(Info info) { if (Size.Type.MAX.equals(params.getSize().getType())) { diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResource.java index fa5db95d7..7384a5aa8 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResource.java @@ -1,11 +1,7 @@ package edu.illinois.library.cantaloupe.resource.iiif.v3; -import java.time.Instant; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Set; import edu.illinois.library.cantaloupe.http.Method; @@ -16,6 +12,7 @@ import edu.illinois.library.cantaloupe.resource.ResourceException; import edu.illinois.library.cantaloupe.resource.Route; import edu.illinois.library.cantaloupe.resource.InformationRequestHandler; +import edu.illinois.library.cantaloupe.source.StatResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,6 +68,14 @@ public boolean authorize() throws Exception { } return false; } + + @Override + public void sourceAccessed(StatResult result) { + if (result.getLastModified() != null) { + setLastModifiedHeader(result.getLastModified()); + } + } + @Override public void knowAvailableOutputFormats(Set formats) { availableOutputFormats.addAll(formats); @@ -97,11 +102,7 @@ private void addHeaders(Info info) { getResponse().setHeader("Content-Type", getNegotiatedContentType()); // Last-Modified if (info.getSerializationTimestamp() != null) { - getResponse().setHeader("Last-Modified", - DateTimeFormatter.RFC_1123_DATE_TIME - .withLocale(Locale.UK) - .withZone(ZoneId.systemDefault()) - .format(info.getSerializationTimestamp())); + setLastModifiedHeader(info.getSerializationTimestamp()); } } diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandlerTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandlerTest.java index db1fce747..7c561f226 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandlerTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandlerTest.java @@ -13,6 +13,7 @@ import edu.illinois.library.cantaloupe.processor.Processor; import edu.illinois.library.cantaloupe.delegate.DelegateProxy; import edu.illinois.library.cantaloupe.processor.SourceFormatException; +import edu.illinois.library.cantaloupe.source.StatResult; import edu.illinois.library.cantaloupe.test.BaseTest; import edu.illinois.library.cantaloupe.test.TestUtil; import edu.illinois.library.cantaloupe.test.WebServer; @@ -103,6 +104,7 @@ void withDelegateProxyWithNullRequestContext() { private static class IntrospectiveCallback implements ImageRequestHandler.Callback { private boolean isPreAuthorizeCalled, isAuthorizeCalled, + isSourceAccessedCalled, isWillStreamImageFromDerivativeCacheCalled, isInfoAvailableCalled, isWillProcessImageCalled; @@ -118,6 +120,11 @@ public boolean authorize() { return true; } + @Override + public void sourceAccessed(StatResult result) { + isSourceAccessedCalled = true; + } + @Override public void willStreamImageFromDerivativeCache() { isWillStreamImageFromDerivativeCacheCalled = true; @@ -186,6 +193,32 @@ void handleCallsAuthorizationCallback() throws Exception { } } + @Test + void handleCallsSourceAccessedCallback() throws Exception { + { // Configure the application. + final Configuration config = Configuration.getInstance(); + config.setProperty(Key.CACHE_SERVER_RESOLVE_FIRST, false); + config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); + config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, + TestUtil.getImagesPath() + "/"); + } + + // Configure the request. + final OperationList opList = new OperationList(); + opList.setIdentifier(new Identifier("jpg-rgb-64x48x8.jpg")); + opList.add(new Encode(Format.get("jpg"))); + + final IntrospectiveCallback callback = new IntrospectiveCallback(); + try (ImageRequestHandler handler = ImageRequestHandler.builder() + .withCallback(callback) + .withOperationList(opList) + .build(); + OutputStream outputStream = OutputStream.nullOutputStream()) { + handler.handle(outputStream); + assertTrue(callback.isSourceAccessedCalled); + } + } + @Test void handleCallsCacheStreamingCallback() throws Exception { { // Configure the application. @@ -393,6 +426,9 @@ public boolean authorize() { return true; } @Override + public void sourceAccessed(StatResult result) { + } + @Override public void willStreamImageFromDerivativeCache() { } @Override @@ -435,6 +471,9 @@ public boolean authorize() { return false; } @Override + public void sourceAccessed(StatResult result) { + } + @Override public void willStreamImageFromDerivativeCache() { } @Override diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/InformationRequestHandlerTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/InformationRequestHandlerTest.java index 9778cc096..cd6c2c319 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/InformationRequestHandlerTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/InformationRequestHandlerTest.java @@ -6,6 +6,7 @@ import edu.illinois.library.cantaloupe.config.Key; import edu.illinois.library.cantaloupe.image.*; import edu.illinois.library.cantaloupe.delegate.DelegateProxy; +import edu.illinois.library.cantaloupe.source.StatResult; import edu.illinois.library.cantaloupe.test.BaseTest; import edu.illinois.library.cantaloupe.test.TestUtil; import org.junit.jupiter.api.Nested; @@ -40,6 +41,7 @@ void testBuildWithDelegateProxyButNoRequestContextSet() { private static class IntrospectiveCallback implements InformationRequestHandler.Callback { private boolean isAuthorizeCalled, + isSourceAccessedCalled, isKnowAvailableOutputFormatsCalled; @Override @@ -48,6 +50,11 @@ public boolean authorize() { return true; } + @Override + public void sourceAccessed(StatResult result) { + isSourceAccessedCalled = true; + } + @Override public void knowAvailableOutputFormats(Set formats) { isKnowAvailableOutputFormatsCalled = true; @@ -74,6 +81,25 @@ void testHandleCallsAuthorizationCallback() throws Exception { } } + @Test + void testHandleCallsSourceAccessedCallback() throws Exception { + { // Configure the application. + final Configuration config = Configuration.getInstance(); + config.setProperty(Key.SOURCE_STATIC, "FilesystemSource"); + config.setProperty(Key.FILESYSTEMSOURCE_PATH_PREFIX, + TestUtil.getImagesPath() + "/"); + } + + final IntrospectiveCallback callback = new IntrospectiveCallback(); + try (InformationRequestHandler handler = InformationRequestHandler.builder() + .withCallback(callback) + .withIdentifier(new Identifier("jpg-rgb-64x48x8.jpg")) + .build()) { + handler.handle(); + assertTrue(callback.isSourceAccessedCalled); + } + } + @Test void testHandleCallsAvailableOutputFormatsCallback() throws Exception { { // Configure the application. @@ -225,6 +251,9 @@ public boolean authorize() { return false; } @Override + public void sourceAccessed(StatResult result) { + } + @Override public void knowAvailableOutputFormats(Set availableOutputFormats) { } }) diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/ImageAPIResourceTester.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/ImageAPIResourceTester.java index 5162b9c96..942dc2d38 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/ImageAPIResourceTester.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/ImageAPIResourceTester.java @@ -30,7 +30,12 @@ import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileTime; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.temporal.TemporalAccessor; import java.util.Iterator; +import java.util.Locale; import static edu.illinois.library.cantaloupe.test.Assert.HTTPAssert.*; import static edu.illinois.library.cantaloupe.test.Assert.PathAssert.*; @@ -301,7 +306,14 @@ public void testLastModifiedHeaderWhenDerivativeCacheIsEnabled(URI uri) client.send(); // request it again to get the Last-Modified header Response response = client.send(); - assertNotNull(response.getHeaders().getFirstValue("Last-Modified")); + String value = response.getHeaders().getFirstValue("Last-Modified"); + TemporalAccessor ta = DateTimeFormatter.RFC_1123_DATE_TIME + .withLocale(Locale.UK) + .withZone(ZoneId.systemDefault()) + .parse(value); + Instant instant = Instant.from(ta); + // assert that the header value is less than 2 seconds in the past + assertTrue(Instant.now().getEpochSecond() - instant.getEpochSecond() < 2); } finally { client.stop(); } diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageResourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageResourceTest.java index 40eb7924d..b83580287 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageResourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/ImageResourceTest.java @@ -544,7 +544,7 @@ void testGETResponseHeaders() throws Exception { client = newClient("/" + IMAGE + "/full/full/0/color.jpg"); Response response = client.send(); Headers headers = response.getHeaders(); - assertEquals(8, headers.size()); + assertEquals(9, headers.size()); // Access-Control-Allow-Origin assertEquals("*", headers.getFirstValue("Access-Control-Allow-Origin")); @@ -554,6 +554,8 @@ void testGETResponseHeaders() throws Exception { assertEquals("image/jpeg", headers.getFirstValue("Content-Type")); // Date assertNotNull(headers.getFirstValue("Date")); + // Last-Modified + assertNotNull(headers.getFirstValue("Last-Modified")); // Link assertTrue(headers.getFirstValue("Link").contains("://")); // Server diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResourceTest.java index a78c4db13..f953b8ee1 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v1/InformationResourceTest.java @@ -541,7 +541,7 @@ void testGETResponseHeaders() throws Exception { client = newClient("/" + IMAGE + "/info.json"); Response response = client.send(); Headers headers = response.getHeaders(); - assertEquals(8, headers.size()); + assertEquals(9, headers.size()); // Access-Control-Allow-Origin assertEquals("*", headers.getFirstValue("Access-Control-Allow-Origin")); @@ -552,8 +552,8 @@ void testGETResponseHeaders() throws Exception { headers.getFirstValue("Content-Type"))); // Date assertNotNull(headers.getFirstValue("Date")); - // Last-Modified (only present for cached infos, tested separately) - assertNull(headers.getFirstValue("Last-Modified")); + // Last-Modified + assertNotNull(headers.getFirstValue("Last-Modified")); // Link assertTrue(headers.getFirstValue("Link").contains("://")); // Server diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageResourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageResourceTest.java index d2f6008ea..e8c2c9ee7 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageResourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/ImageResourceTest.java @@ -646,7 +646,7 @@ void testGETResponseHeaders() throws Exception { client = newClient("/" + IMAGE + "/full/full/0/color.jpg"); Response response = client.send(); Headers headers = response.getHeaders(); - assertEquals(8, headers.size()); + assertEquals(9, headers.size()); // Access-Control-Allow-Origin assertEquals("*", headers.getFirstValue("Access-Control-Allow-Origin")); @@ -656,6 +656,8 @@ void testGETResponseHeaders() throws Exception { assertEquals("image/jpeg", headers.getFirstValue("Content-Type")); // Date assertNotNull(headers.getFirstValue("Date")); + // Last-Modified + assertNotNull(headers.getFirstValue("Last-Modified")); // Link assertTrue(headers.getFirstValue("Link").contains("://")); // Server diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResourceTest.java index d2b11caed..ad65dfa3d 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v2/InformationResourceTest.java @@ -556,7 +556,7 @@ void testGETResponseHeaders() throws Exception { client = newClient("/" + IMAGE + "/info.json"); Response response = client.send(); Headers headers = response.getHeaders(); - assertEquals(7, headers.size()); + assertEquals(8, headers.size()); // Access-Control-Allow-Origin assertEquals("*", headers.getFirstValue("Access-Control-Allow-Origin")); @@ -567,8 +567,8 @@ void testGETResponseHeaders() throws Exception { headers.getFirstValue("Content-Type"))); // Date assertNotNull(headers.getFirstValue("Date")); - // Last-Modified (only present for cached infos, tested separately) - assertNull(headers.getFirstValue("Last-Modified")); + // Last-Modified + assertNotNull(headers.getFirstValue("Last-Modified")); // Server assertNotNull(headers.getFirstValue("Server")); // Vary diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageResourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageResourceTest.java index 2b834f61b..d5d0a00b7 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageResourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/ImageResourceTest.java @@ -596,7 +596,7 @@ void testGETResponseHeaders() throws Exception { client = newClient("/" + IMAGE + "/full/max/0/color.jpg"); Response response = client.send(); Headers headers = response.getHeaders(); - assertEquals(8, headers.size()); + assertEquals(9, headers.size()); // Access-Control-Allow-Origin assertEquals("*", headers.getFirstValue("Access-Control-Allow-Origin")); @@ -606,6 +606,8 @@ void testGETResponseHeaders() throws Exception { assertEquals("image/jpeg", headers.getFirstValue("Content-Type")); // Date assertNotNull(headers.getFirstValue("Date")); + // Last-Modified + assertNotNull(headers.getFirstValue("Last-Modified")); // Link assertTrue(headers.getFirstValue("Link").contains("://")); // Server diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResourceTest.java index 8618c03cb..d03348521 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/iiif/v3/InformationResourceTest.java @@ -531,7 +531,7 @@ void testGETResponseHeaders() throws Exception { client = newClient("/" + IMAGE + "/info.json"); Response response = client.send(); Headers headers = response.getHeaders(); - assertEquals(7, headers.size()); + assertEquals(8, headers.size()); // Access-Control-Allow-Origin assertEquals("*", headers.getFirstValue("Access-Control-Allow-Origin")); @@ -542,8 +542,8 @@ void testGETResponseHeaders() throws Exception { headers.getFirstValue("Content-Type"))); // Date assertNotNull(headers.getFirstValue("Date")); - // Last-Modified (only present for cached infos, tested separately) - assertNull(headers.getFirstValue("Last-Modified")); + // Last-Modified + assertNotNull(headers.getFirstValue("Last-Modified")); // Server assertNotNull(headers.getFirstValue("Server")); // Vary From 21057cf72071840e2a984450bf8845c6e6490ec9 Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Fri, 25 Mar 2022 13:37:37 -0500 Subject: [PATCH 033/106] Remove HttpSource.proxy.http.enabled and rename HttpSource.proxy.http.server to host (#502) --- CHANGES.md | 25 +++++++--- UPGRADING.md | 2 + cantaloupe.properties.sample | 9 ++-- .../library/cantaloupe/config/Key.java | 5 +- .../library/cantaloupe/source/HttpSource.java | 50 +++++++------------ src/main/resources/admin.vm | 28 +++++++++++ .../resource/admin/AdminResourceUITest.java | 6 +++ .../source/HTTPStreamFactoryTest.java | 46 +++++++++++++++++ .../cantaloupe/source/HttpSourceTest.java | 45 ++++++++++++----- 9 files changed, 158 insertions(+), 58 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2a5e83238..6146b708b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,23 +1,36 @@ # Change Log -# 6.0 +## 6.0 + +### Endpoints * Image and information responses include a `Last-Modified` header when possible. * The health endpoint is enabled via `endpoint.health.enabled` rather than `endpoint.api.enabled`. +* Added an HTTP API method to purge all infos from the derivative cache. +* Added a configuration option to automatically purge source-cached images + whose format cannot be inferred. + +### Sources + +* HttpSource supports a client HTTP proxy. (Thanks to @mightymax and + @mlindeman) * HttpSource can be configured to send a ranged GET request instead of a HEAD request, enabling it to work with pre-signed URLs that do not allow HEAD requests. * S3Source supports multiple endpoints when using ScriptLookupStrategy. -* Added a configuration option to automatically purge source-cached images - whose format cannot be inferred. -* Added an HTTP API method to purge all infos from the derivative cache. + +### Caches + +* S3Cache uses multipart uploads, which reduces memory usage when caching + derivatives larger than 5 MB. + +### Delegate Script + * The delegate script pathname can be set using the `-Dcantaloupe.delegate_script` VM argument, which takes precedence over the `delegate_script.pathname` configuration key. -* S3Cache uses multipart uploads, which reduces memory usage when caching - derivatives larger than 5 MB. * The delegate script's `metadata` context key contains a new field, `xmp_elements`, that provides a high-level key-value view of the XMP data. diff --git a/UPGRADING.md b/UPGRADING.md index 58261b77b..3e38801c7 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -7,6 +7,8 @@ current version. 1. Add the following keys from the sample configuration: * `endpoint.health.enabled` + * `HttpSource.proxy.http.host` + * `HttpSource.proxy.http.port` * `HttpSource.BasicLookupStrategy.send_head_requests` * `processor.purge_incompatible_from_source_cache` 2. Add the following methods from the sample delegate script: diff --git a/cantaloupe.properties.sample b/cantaloupe.properties.sample index e5450d649..e205ad760 100644 --- a/cantaloupe.properties.sample +++ b/cantaloupe.properties.sample @@ -175,6 +175,10 @@ HttpSource.allow_insecure = false # Request timeout in seconds. HttpSource.request_timeout = +# !! Client HTTP proxy. +HttpSource.proxy.http.host = +HttpSource.proxy.http.port = + # Tells HttpSource how to look up resources. Allowed values are # `BasicLookupStrategy` and `ScriptLookupStrategy`. ScriptLookupStrategy # uses a delegate method for dynamic lookups; see the user manual. @@ -212,11 +216,6 @@ HttpSource.chunking.cache.enabled = true # Max per-request chunk cache size. HttpSource.chunking.cache.max_size = 5M -# Enable HTTP Proxy for HttpSource -HttpSource.proxy.http.enabled = false -HttpSource.proxy.http.server = proxy.example.com -HttpSource.proxy.http.port = 8080 - #---------------------------------------- # S3Source #---------------------------------------- diff --git a/src/main/java/edu/illinois/library/cantaloupe/config/Key.java b/src/main/java/edu/illinois/library/cantaloupe/config/Key.java index a315294a9..2ac19ce09 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/config/Key.java +++ b/src/main/java/edu/illinois/library/cantaloupe/config/Key.java @@ -108,6 +108,8 @@ public enum Key { HTTPSOURCE_CHUNK_SIZE("HttpSource.chunking.chunk_size"), HTTPSOURCE_CHUNK_CACHE_ENABLED("HttpSource.chunking.cache.enabled"), HTTPSOURCE_CHUNK_CACHE_MAX_SIZE("HttpSource.chunking.cache.max_size"), + HTTPSOURCE_HTTP_PROXY_HOST("HttpSource.proxy.http.host"), + HTTPSOURCE_HTTP_PROXY_PORT("HttpSource.proxy.http.port"), HTTPSOURCE_LOOKUP_STRATEGY("HttpSource.lookup_strategy"), HTTPSOURCE_REQUEST_TIMEOUT("HttpSource.request_timeout"), HTTPSOURCE_SEND_HEAD_REQUESTS("HttpSource.BasicLookupStrategy.send_head_requests"), @@ -120,9 +122,6 @@ public enum Key { HTTPS_KEY_STORE_PATH("https.key_store_path"), HTTPS_KEY_STORE_TYPE("https.key_store_type"), HTTPS_PORT("https.port"), - HTTPSOURCE_HTTP_PROXY_ENABLED("HttpSource.proxy.http.enabled"), - HTTPSOURCE_HTTP_PROXY_SERVER("HttpSource.proxy.http.server"), - HTTPSOURCE_HTTP_PROXY_PORT("HttpSource.proxy.http.port"), IIIF_1_ENDPOINT_ENABLED("endpoint.iiif.1.enabled"), IIIF_2_ENDPOINT_ENABLED("endpoint.iiif.2.enabled"), IIIF_3_ENDPOINT_ENABLED("endpoint.iiif.3.enabled"), diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java b/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java index 5208a6314..e9c2af51d 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java @@ -19,10 +19,10 @@ import javax.net.ssl.X509TrustManager; import javax.script.ScriptException; import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Proxy; import java.net.URI; import java.net.URISyntaxException; -import java.net.Proxy; -import java.net.InetSocketAddress; import java.nio.file.AccessDeniedException; import java.nio.file.NoSuchFileException; import java.security.KeyManagementException; @@ -410,27 +410,24 @@ static synchronized OkHttpClient getHTTPClient() { .connectTimeout(getRequestTimeout().getSeconds(), TimeUnit.SECONDS) .readTimeout(getRequestTimeout().getSeconds(), TimeUnit.SECONDS) .writeTimeout(getRequestTimeout().getSeconds(), TimeUnit.SECONDS); - final Configuration config = Configuration.getInstance(); - - final boolean httpProxyEnabled = config.getBoolean( - Key.HTTPSOURCE_HTTP_PROXY_ENABLED, false); - if (httpProxyEnabled) { - final String httpProxyServer = config.getString(Key.HTTPSOURCE_HTTP_PROXY_SERVER, ""); - if (httpProxyServer == "") { - throw new RuntimeException("proxy server setting HttpSource.proxy.http.server should not be empty"); - } - final int httpProxyPort = config.getInt(Key.HTTPSOURCE_HTTP_PROXY_PORT, 8080); - - LOGGER.trace("Using HTTP Proxy at server {} on port {}", httpProxyServer, httpProxyPort); - Proxy httpProxy = new Proxy(Proxy.Type.HTTP,new InetSocketAddress(httpProxyServer, httpProxyPort)); - builder.proxy(httpProxy); - } - final boolean allowInsecure = config.getBoolean( - Key.HTTPSOURCE_ALLOW_INSECURE, false); + final String proxyHost = + config.getString(Key.HTTPSOURCE_HTTP_PROXY_HOST, ""); + if (!proxyHost.isBlank()) { + final int proxyPort = + config.getInt(Key.HTTPSOURCE_HTTP_PROXY_PORT); + if (proxyPort == 0) { + throw new RuntimeException("Proxy port setting " + + Key.HTTPSOURCE_HTTP_PROXY_PORT + " must be set"); + } + LOGGER.debug("Using HTTP proxy: {}:{}", proxyHost, proxyPort); + Proxy httpProxy = new Proxy(Proxy.Type.HTTP, + new InetSocketAddress(proxyHost, proxyPort)); + builder.proxy(httpProxy); + } - if (allowInsecure) { + if (config.getBoolean(Key.HTTPSOURCE_ALLOW_INSECURE, false)) { try { X509TrustManager[] tm = new X509TrustManager[]{ new X509TrustManager() { @@ -470,17 +467,6 @@ private static Duration getRequestTimeout() { return Duration.ofSeconds(timeout); } - static String getUserAgent() { - return String.format("%s/%s (%s/%s; java/%s; %s/%s)", - HttpSource.class.getSimpleName(), - Application.getVersion(), - Application.getName(), - Application.getVersion(), - System.getProperty("java.version"), - System.getProperty("os.name"), - System.getProperty("os.version")); - } - /** * @see #request(HTTPRequestInfo, String, Map) */ @@ -505,7 +491,7 @@ static Response request(HTTPRequestInfo requestInfo, Request.Builder builder = new Request.Builder() .method(method, null) .url(requestInfo.getURI()) - .addHeader("User-Agent", getUserAgent()); + .addHeader("User-Agent", USER_AGENT); // Add credentials. if (requestInfo.getUsername() != null && requestInfo.getSecret() != null) { diff --git a/src/main/resources/admin.vm b/src/main/resources/admin.vm index 45a46327e..8890f1648 100644 --- a/src/main/resources/admin.vm +++ b/src/main/resources/admin.vm @@ -1721,6 +1721,34 @@ data-requires-restart="false">
      + Proxy Host + ? + + +
      + Proxy Port + ? + + +
      Lookup Strategy 1000); + } + @Test void newInputStreamSendsCustomHeaders() throws Exception { server.setHandler(new DefaultHandler() { @@ -138,6 +162,28 @@ void newSeekableStreamWhenChunkingIsDisabled() throws Exception { } } + @Disabled + @Test + void newSeekableStreamWithProxy() throws Exception { + final int proxyPort = SocketUtils.getOpenPort(); + + // Set up the proxy + // TODO: write this + + // Set up HttpSource + final var config = Configuration.getInstance(); + config.setProperty(Key.HTTPSOURCE_HTTP_PROXY_HOST, "127.0.0.1"); + config.setProperty(Key.HTTPSOURCE_HTTP_PROXY_PORT, proxyPort); + + int length = 0; + try (ImageInputStream is = newInstance(true).newSeekableStream()) { + while (is.read() != -1) { + length++; + } + } + assertTrue(length > 1000); + } + @Test void newSeekableStreamSendsCustomHeaders() throws Exception { server.setHandler(new DefaultHandler() { diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java index ccbfda23b..a30046c2a 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java @@ -9,10 +9,12 @@ import edu.illinois.library.cantaloupe.delegate.DelegateProxy; import edu.illinois.library.cantaloupe.test.TestUtil; import edu.illinois.library.cantaloupe.test.WebServer; +import edu.illinois.library.cantaloupe.util.SocketUtils; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.DefaultHandler; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import jakarta.servlet.http.HttpServletRequest; @@ -117,16 +119,16 @@ void useScriptLookupStrategy() { "ScriptLookupStrategy"); } - /* checkAccess() */ + /* stat() */ @Test - void testCheckAccessUsingBasicLookupStrategyWithPresentUnreadableImage() + void testStatUsingBasicLookupStrategyWithPresentUnreadableImage() throws Exception { doTestCheckAccessWithPresentUnreadableImage(new Identifier("gif")); } @Test - void testCheckAccessUsingScriptLookupStrategyWithPresentReadableImage() + void testStatUsingScriptLookupStrategyWithPresentReadableImage() throws Exception { useScriptLookupStrategy(); Identifier identifier = new Identifier(getServerURI() + "/" + @@ -135,7 +137,7 @@ void testCheckAccessUsingScriptLookupStrategyWithPresentReadableImage() } @Test - void testCheckAccessUsingScriptLookupStrategyWithMissingImage() + void testStatUsingScriptLookupStrategyWithMissingImage() throws Exception { useScriptLookupStrategy(); Identifier identifier = new Identifier(getServerURI() + "/bogus"); @@ -143,7 +145,7 @@ void testCheckAccessUsingScriptLookupStrategyWithMissingImage() } @Test - void testCheckAccessUsingScriptLookupStrategyWithPresentUnreadableImage() + void testStatUsingScriptLookupStrategyWithPresentUnreadableImage() throws Exception { useScriptLookupStrategy(); Identifier identifier = new Identifier(getServerURI() + "/gif"); @@ -202,7 +204,7 @@ private void doTestCheckAccessWithMissingImage(Identifier identifier) } @Test - void testCheckAccessUsingScriptLookupStrategyWithValidAuthentication() + void testStatUsingScriptLookupStrategyWithValidAuthentication() throws Exception { useScriptLookupStrategy(); @@ -220,7 +222,7 @@ void testCheckAccessUsingScriptLookupStrategyWithValidAuthentication() } @Test - void testCheckAccessUsingScriptLookupStrategyWithInvalidAuthentication() + void testStatUsingScriptLookupStrategyWithInvalidAuthentication() throws Exception { useScriptLookupStrategy(); @@ -238,7 +240,7 @@ void testCheckAccessUsingScriptLookupStrategyWithInvalidAuthentication() } @Test - void testCheckAccessWith403Response() throws Exception { + void testStatWith403Response() throws Exception { server.setHandler(new DefaultHandler() { @Override public void handle(String target, @@ -261,7 +263,7 @@ public void handle(String target, } @Test - void testCheckAccessWith500Response() throws Exception { + void testStatWith500Response() throws Exception { server.setHandler(new DefaultHandler() { @Override public void handle(String target, @@ -283,8 +285,27 @@ public void handle(String target, } } + @Disabled @Test - void testCheckAccessSendsUserAgentHeader() throws Exception { + void testStatUsingProxy() throws Exception { + server.start(); + + final int proxyPort = SocketUtils.getOpenPort(); + + // Set up the proxy + // TODO; write this + + // Set up HttpSource + final Configuration config = Configuration.getInstance(); + config.setProperty(Key.HTTPSOURCE_HTTP_PROXY_HOST, "127.0.0.1"); + config.setProperty(Key.HTTPSOURCE_HTTP_PROXY_PORT, proxyPort); + + // Expect no exception + instance.stat(); + } + + @Test + void testStatSendsUserAgentHeader() throws Exception { server.setHandler(new DefaultHandler() { @Override public void handle(String target, @@ -309,7 +330,7 @@ public void handle(String target, } @Test - void testCheckAccessSendsCustomHeaders() throws Exception { + void testStatSendsCustomHeaders() throws Exception { useScriptLookupStrategy(); server.setHandler(new DefaultHandler() { @@ -335,7 +356,7 @@ public void handle(String target, } @Test - void testCheckAccessWithMalformedURI() throws Exception { + void testStatWithMalformedURI() throws Exception { server.start(); Configuration config = Configuration.getInstance(); From c2a9cb5a815c0a49774fd36fa3a71d499cadb60a Mon Sep 17 00:00:00 2001 From: Diego Pino Navarro Date: Fri, 11 Aug 2023 11:09:23 -0400 Subject: [PATCH 034/106] Fix Virtual Cropping detection condition to take both Y and X in account --- .../cantaloupe/processor/codec/jpeg/TurboJPEGImageWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriter.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriter.java index c87fc5f41..b9ac8f411 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriter.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriter.java @@ -155,7 +155,7 @@ public void write(BufferedImage image, // Also, JPEG doesn't support alpha, so we have to remove that, // otherwise readers will interpret as CMYK. if (image.getRaster().getSampleModelTranslateX() < 0 || - image.getRaster().getSampleModelTranslateX() < 0) { + image.getRaster().getSampleModelTranslateY() < 0) { BufferedImage newImage = new BufferedImage( image.getWidth(), image.getHeight(), BufferedImage.TYPE_INT_RGB); From 678b03f1b0577921dbd7f4dcd3ec15f1b2ce1747 Mon Sep 17 00:00:00 2001 From: Daniel Pierce Date: Mon, 26 Feb 2024 23:13:41 -0500 Subject: [PATCH 035/106] Use gray subsampling for grayscale images Fixes error when source image is grayscale jp2 --- .../processor/codec/jpeg/TurboJPEGImageWriter.java | 5 +++++ .../codec/jpeg/TurboJPEGImageWriterTest.java | 13 ++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriter.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriter.java index c87fc5f41..dd28fa90a 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriter.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriter.java @@ -167,6 +167,11 @@ public void write(BufferedImage image, image = Java2DUtil.removeAlpha(image, bgColor); image = Java2DUtil.convertCustomToRGB(image); + // Gray subsampling required to handle grayscale input + if (image.getType() == BufferedImage.TYPE_BYTE_GRAY) { + setSubsampling(TJ.SAMP_GRAY); + } + try (TJCompressor tjc = new TJCompressor()) { tjc.setSubsamp(subsampling); tjc.setJPEGQuality(quality); diff --git a/src/test/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriterTest.java b/src/test/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriterTest.java index 74a752a69..142fb37fc 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriterTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriterTest.java @@ -164,6 +164,17 @@ public void testWriteWithBufferedImage() throws Exception { } } + @Test + public void testWriteWithGrayBufferedImage() throws Exception { + BufferedImage image = new BufferedImage(50, 50, + BufferedImage.TYPE_BYTE_GRAY); + + try (ByteArrayOutputStream os = new ByteArrayOutputStream()) { + instance.write(image, os); + assertDimensions(os, image.getWidth(), image.getHeight()); + } + } + @Test public void testWriteWithBufferedImageWithBackgroundColor() throws Exception { @@ -184,4 +195,4 @@ public void testWriteWithBufferedImageWithBackgroundColor() } } -} \ No newline at end of file +} From 37f232ffe290fe0b02ffd9e58dd52c62541757c7 Mon Sep 17 00:00:00 2001 From: Justin Coyne Date: Tue, 18 Jun 2024 16:50:52 -0500 Subject: [PATCH 036/106] Be explicit about the usage of the properties file --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1a2e84a7b..3b53f62d9 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,8 @@ ### Command line -* `mvn clean compile exec:java -Dcantaloupe.config=...` will build and run the +* `cp cantaloupe.properties.sample cantaloupe.properties` then edit cantaloupe.properties per your needs. +* `mvn clean compile exec:java -Dcantaloupe.config=cantaloupe.properties` will build and run the project using the embedded web server listening on the port(s) specified in `cantaloupe.properties`. * `mvn clean package -DskipTests` will build a release JAR in the `target` @@ -25,7 +26,7 @@ 1. Add a new run configuration using the "Java Application" template or similar. 2. Set the main class to `edu.illinois.library.cantaloupe.StandaloneEntry` and - add the `-Dcantaloupe.config=...` VM option. + add the `-Dcantaloupe.config=cantaloupe.properties` VM option. ## Test From cf4edf950e42b6f4635b6dcefcc1484e21f82d15 Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Fri, 25 Mar 2022 14:16:15 -0500 Subject: [PATCH 037/106] Add --build arguments to docker compose invocations --- .github/workflows/ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a07fb0d62..24ed4da9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,24 +13,24 @@ jobs: uses: actions/checkout@v2 - name: Test in Linux JDK 11 if: matrix.os == 'ubuntu-latest' && matrix.java == 'jdk11' - run: docker-compose -f docker/Linux-JDK11/docker-compose.yml up --exit-code-from cantaloupe + run: docker-compose -f docker/Linux-JDK11/docker-compose.yml up --build --exit-code-from cantaloupe - name: Test in Linux JDK 15 if: matrix.os == 'ubuntu-latest' && matrix.java == 'jdk15' - run: docker-compose -f docker/Linux-JDK15/docker-compose.yml up --exit-code-from cantaloupe + run: docker-compose -f docker/Linux-JDK15/docker-compose.yml up --build --exit-code-from cantaloupe - name: Test in Linux JDK 16 if: matrix.os == 'ubuntu-latest' && matrix.java == 'jdk16' - run: docker-compose -f docker/Linux-JDK16/docker-compose.yml up --exit-code-from cantaloupe + run: docker-compose -f docker/Linux-JDK16/docker-compose.yml up --build --exit-code-from cantaloupe - name: Test in Linux GraalVM if: matrix.os == 'ubuntu-latest' && matrix.java == 'graalvm' - run: docker-compose -f docker/Linux-GraalVM20/docker-compose.yml up --exit-code-from cantaloupe + run: docker-compose -f docker/Linux-GraalVM20/docker-compose.yml up --build --exit-code-from cantaloupe - name: Test in Windows JDK 11 if: matrix.os == 'windows-latest' && matrix.java == 'jdk11' - run: docker-compose -f docker/Windows-JDK11/docker-compose.yml up --exit-code-from cantaloupe + run: docker-compose -f docker/Windows-JDK11/docker-compose.yml up --build --exit-code-from cantaloupe - name: Test in Windows JDK 15 if: matrix.os == 'windows-latest' && matrix.java == 'jdk15' - run: docker-compose -f docker/Windows-JDK15/docker-compose.yml up --exit-code-from cantaloupe + run: docker-compose -f docker/Windows-JDK15/docker-compose.yml up --build --exit-code-from cantaloupe - name: Test in Windows JDK 16 if: matrix.os == 'windows-latest' && matrix.java == 'jdk16' - run: docker-compose -f docker/Windows-JDK16/docker-compose.yml up --exit-code-from cantaloupe + run: docker-compose -f docker/Windows-JDK16/docker-compose.yml up --build --exit-code-from cantaloupe # TODO: Windows+GraalVM From 2d26e014b91aad6015285a4c224b725996348249 Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Fri, 25 Mar 2022 15:54:23 -0500 Subject: [PATCH 038/106] Add an --add-opens argument to the surefire invocation to support JAI in Java 16+ --- pom.xml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 6f0550f0c..9c7b9ce25 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ 11.0.5 9.2.17.0 1.2.8 - 3.0.0-M3 + 3.0.0-M5 @@ -421,7 +421,8 @@ random false - --illegal-access=permit + + --illegal-access=permit --add-opens java.desktop/sun.awt.image=ALL-UNNAMED @@ -557,6 +558,8 @@ random false + + --add-opens java.desktop/sun.awt.image=ALL-UNNAMED AzureStorage*Test FfmpegProcessorTest @@ -586,6 +589,8 @@ random false + + --add-opens java.desktop/sun.awt.image=ALL-UNNAMED Azure*Test From 3ccfc297e9d4aa94325fbbff398cb19a219934d3 Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Fri, 25 Mar 2022 17:20:10 -0500 Subject: [PATCH 039/106] Replace JDK 15 & 16 with 18 in CI and update Windows containers to Server Core 2022 --- .github/workflows/ci.yml | 20 +++---- docker/Linux-JDK15/Dockerfile | 57 ------------------- docker/Linux-JDK15/docker-compose.yml | 21 ------- .../{Linux-JDK16 => Linux-JDK18}/Dockerfile | 16 +++--- .../docker-compose.yml | 2 +- docker/Windows-JDK11/Dockerfile | 2 +- docker/Windows-JDK15/Dockerfile | 28 --------- docker/Windows-JDK15/Dockerfile-minio | 10 ---- docker/Windows-JDK15/docker-compose.yml | 19 ------- .../Dockerfile | 4 +- .../Dockerfile-minio | 2 +- .../docker-compose.yml | 4 +- 12 files changed, 22 insertions(+), 163 deletions(-) delete mode 100644 docker/Linux-JDK15/Dockerfile delete mode 100644 docker/Linux-JDK15/docker-compose.yml rename docker/{Linux-JDK16 => Linux-JDK18}/Dockerfile (76%) rename docker/{Linux-JDK16 => Linux-JDK18}/docker-compose.yml (89%) delete mode 100644 docker/Windows-JDK15/Dockerfile delete mode 100644 docker/Windows-JDK15/Dockerfile-minio delete mode 100644 docker/Windows-JDK15/docker-compose.yml rename docker/{Windows-JDK16 => Windows-JDK18}/Dockerfile (89%) rename docker/{Windows-JDK16 => Windows-JDK18}/Dockerfile-minio (81%) rename docker/{Windows-JDK16 => Windows-JDK18}/docker-compose.yml (77%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 24ed4da9a..e9e6dfe5a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest] - java: [jdk11, jdk15, graalvm] + java: [jdk11, jdk18, graalvm] fail-fast: false steps: - name: Check out the repository @@ -14,23 +14,17 @@ jobs: - name: Test in Linux JDK 11 if: matrix.os == 'ubuntu-latest' && matrix.java == 'jdk11' run: docker-compose -f docker/Linux-JDK11/docker-compose.yml up --build --exit-code-from cantaloupe - - name: Test in Linux JDK 15 - if: matrix.os == 'ubuntu-latest' && matrix.java == 'jdk15' - run: docker-compose -f docker/Linux-JDK15/docker-compose.yml up --build --exit-code-from cantaloupe - - name: Test in Linux JDK 16 - if: matrix.os == 'ubuntu-latest' && matrix.java == 'jdk16' - run: docker-compose -f docker/Linux-JDK16/docker-compose.yml up --build --exit-code-from cantaloupe + - name: Test in Linux JDK 18 + if: matrix.os == 'ubuntu-latest' && matrix.java == 'jdk18' + run: docker-compose -f docker/Linux-JDK18/docker-compose.yml up --build --exit-code-from cantaloupe - name: Test in Linux GraalVM if: matrix.os == 'ubuntu-latest' && matrix.java == 'graalvm' run: docker-compose -f docker/Linux-GraalVM20/docker-compose.yml up --build --exit-code-from cantaloupe - name: Test in Windows JDK 11 if: matrix.os == 'windows-latest' && matrix.java == 'jdk11' run: docker-compose -f docker/Windows-JDK11/docker-compose.yml up --build --exit-code-from cantaloupe - - name: Test in Windows JDK 15 - if: matrix.os == 'windows-latest' && matrix.java == 'jdk15' - run: docker-compose -f docker/Windows-JDK15/docker-compose.yml up --build --exit-code-from cantaloupe - - name: Test in Windows JDK 16 - if: matrix.os == 'windows-latest' && matrix.java == 'jdk16' - run: docker-compose -f docker/Windows-JDK16/docker-compose.yml up --build --exit-code-from cantaloupe + - name: Test in Windows JDK 18 + if: matrix.os == 'windows-latest' && matrix.java == 'jdk18' + run: docker-compose -f docker/Windows-JDK18/docker-compose.yml up --build --exit-code-from cantaloupe # TODO: Windows+GraalVM diff --git a/docker/Linux-JDK15/Dockerfile b/docker/Linux-JDK15/Dockerfile deleted file mode 100644 index d8ffc2296..000000000 --- a/docker/Linux-JDK15/Dockerfile +++ /dev/null @@ -1,57 +0,0 @@ -FROM ubuntu:latest - -ENV JAVA_HOME=/opt/jdk -ENV PATH=$PATH:/opt/jdk/bin -ARG DEBIAN_FRONTEND=noninteractive - -# Install various dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - ffmpeg \ - maven \ - wget \ - libopenjp2-tools \ - liblcms2-dev \ - libpng-dev \ - libzstd-dev \ - libtiff-dev \ - libjpeg-dev \ - zlib1g-dev \ - libwebp-dev \ - libimage-exiftool-perl \ - && rm -rf /var/lib/apt/lists/* - -# Install TurboJpegProcessor dependencies -RUN mkdir -p /opt/libjpeg-turbo/lib -COPY docker/Linux-JDK11/image_files/libjpeg-turbo/lib64 /opt/libjpeg-turbo/lib - -# Install KakaduNativeProcessor dependencies -COPY dist/deps/Linux-x86-64/lib/* /usr/lib/ - -# Install GrokProcessor dependencies -RUN wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/libgrokj2k1_7.6.5-1_amd64.deb \ - && wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/grokj2k-tools_7.6.5-1_amd64.deb \ - && dpkg -i ./libgrokj2k1_7.6.5-1_amd64.deb \ - && dpkg -i --ignore-depends=libjpeg62-turbo ./grokj2k-tools_7.6.5-1_amd64.deb - -# Install OpenJDK -RUN wget -q https://github.com/AdoptOpenJDK/openjdk15-binaries/releases/download/jdk-15.0.1%2B9/OpenJDK15U-jdk_x64_linux_hotspot_15.0.1_9.tar.gz \ - && tar xfz OpenJDK15U-jdk_x64_linux_hotspot_15.0.1_9.tar.gz \ - && mv jdk-15.0.1+9 /opt/jdk - -# A non-root user is needed for some FilesystemSourceTest tests to work. -ARG user=cantaloupe -ARG home=/home/$user -RUN adduser --home $home $user -RUN chown -R $user $home -USER $user -WORKDIR $home - -# Install application dependencies -COPY ./pom.xml pom.xml -RUN mvn --quiet dependency:resolve - -# Copy the code -COPY --chown=cantaloupe docker/Linux-JDK11/image_files/test.properties test.properties -COPY --chown=cantaloupe ./src src - -ENTRYPOINT mvn --batch-mode test -Pfreedeps \ No newline at end of file diff --git a/docker/Linux-JDK15/docker-compose.yml b/docker/Linux-JDK15/docker-compose.yml deleted file mode 100644 index 4b4136275..000000000 --- a/docker/Linux-JDK15/docker-compose.yml +++ /dev/null @@ -1,21 +0,0 @@ -# -# N.B.: docker-compose must be invoked from the project root directory: -# -# docker-compose -f path/to/docker-compose.yml up --exit-code-from cantaloupe -# -version: '3' -services: - cantaloupe: - build: - context: ../../ - dockerfile: $PWD/docker/Linux-JDK15/Dockerfile - minio: - image: minio/minio - environment: - MINIO_ACCESS_KEY: MinioUser - MINIO_SECRET_KEY: OpenSesame - hostname: minio - command: server /data - redis: - image: redis:alpine - hostname: redis \ No newline at end of file diff --git a/docker/Linux-JDK16/Dockerfile b/docker/Linux-JDK18/Dockerfile similarity index 76% rename from docker/Linux-JDK16/Dockerfile rename to docker/Linux-JDK18/Dockerfile index 34e828755..947f543e0 100644 --- a/docker/Linux-JDK16/Dockerfile +++ b/docker/Linux-JDK18/Dockerfile @@ -39,14 +39,14 @@ RUN wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6. && dpkg -i ./libgrokj2k1_7.6.5-1_amd64.deb \ && dpkg -i --ignore-depends=libjpeg62-turbo ./grokj2k-tools_7.6.5-1_amd64.deb \ # Install OpenJDK - && wget -q https://github.com/AdoptOpenJDK/openjdk16-binaries/releases/download/jdk-16.0.1%2B9/OpenJDK16U-jdk_x64_linux_hotspot_16.0.1_9.tar.gz \ - && tar xfz OpenJDK16U-jdk_x64_linux_hotspot_16.0.1_9.tar.gz \ - && mv jdk-16.0.1+9 /opt/jdk \ - # Install Maven (the one in apt is too old for JDK16 as of 2020-05-14) - && wget -q https://mirrors.ocf.berkeley.edu/apache/maven/maven-3/3.8.1/binaries/apache-maven-3.8.1-bin.tar.gz \ - && tar xfz apache-maven-3.8.1-bin.tar.gz \ - && mv apache-maven-3.8.1 /opt/maven \ - && rm apache-maven-3.8.1-bin.tar.gz + && wget -q https://download.java.net/java/GA/jdk18/43f95e8614114aeaa8e8a5fcf20a682d/36/GPL/openjdk-18_linux-x64_bin.tar.gz \ + && tar xfz openjdk-18_linux-x64_bin.tar.gz \ + && mv jdk-18 /opt/jdk \ + # Install a newer Maven than the one in apt + && wget -q https://dlcdn.apache.org/maven/maven-3/3.8.5/binaries/apache-maven-3.8.5-bin.tar.gz \ + && tar xfz apache-maven-3.8.5-bin.tar.gz \ + && mv apache-maven-3.8.5 /opt/maven \ + && rm apache-maven-3.8.5-bin.tar.gz # A non-root user is needed for some FilesystemSourceTest tests to work. ARG user=cantaloupe diff --git a/docker/Linux-JDK16/docker-compose.yml b/docker/Linux-JDK18/docker-compose.yml similarity index 89% rename from docker/Linux-JDK16/docker-compose.yml rename to docker/Linux-JDK18/docker-compose.yml index b6518c805..7b2572bc0 100644 --- a/docker/Linux-JDK16/docker-compose.yml +++ b/docker/Linux-JDK18/docker-compose.yml @@ -8,7 +8,7 @@ services: cantaloupe: build: context: ../../ - dockerfile: $PWD/docker/Linux-JDK16/Dockerfile + dockerfile: $PWD/docker/Linux-JDK18/Dockerfile minio: image: minio/minio environment: diff --git a/docker/Windows-JDK11/Dockerfile b/docker/Windows-JDK11/Dockerfile index 8d81bd6d1..ba1008de8 100644 --- a/docker/Windows-JDK11/Dockerfile +++ b/docker/Windows-JDK11/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/windows/servercore:ltsc2019 +FROM mcr.microsoft.com/windows/servercore:ltsc2022 ENV chocolateyUseWindowsCompression false diff --git a/docker/Windows-JDK15/Dockerfile b/docker/Windows-JDK15/Dockerfile deleted file mode 100644 index 79307a09a..000000000 --- a/docker/Windows-JDK15/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM mcr.microsoft.com/windows/servercore:ltsc2019 - -ENV chocolateyUseWindowsCompression false - -# Install the Chocolatey package manager, which makes it easier to install -# dependencies. -RUN powershell -Command \ - iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); \ - choco feature disable --name showDownloadProgress - -# Install various dependencies -# TODO: openjpeg -RUN choco install -y adoptopenjdk15 maven ffmpeg - -# Install TurboJpegProcessor dependencies TODO: libjpeg-turbo -#RUN mkdir -p /opt/libjpeg-turbo/lib -#COPY docker/Windows10-JDK11/image_files/libjpeg-turbo/lib64 c:\windows\system32 - -# Install KakaduNativeProcessor dependencies -COPY dist/deps/Windows-x86-64/lib/* c:/Windows/System32/ - -# Install application dependencies -COPY pom.xml pom.xml -RUN mvn dependency:resolve - -# Copy the code -COPY docker/Windows-JDK11/image_files/test.properties test.properties -COPY src src \ No newline at end of file diff --git a/docker/Windows-JDK15/Dockerfile-minio b/docker/Windows-JDK15/Dockerfile-minio deleted file mode 100644 index babcdadfd..000000000 --- a/docker/Windows-JDK15/Dockerfile-minio +++ /dev/null @@ -1,10 +0,0 @@ -FROM mcr.microsoft.com/windows/servercore:ltsc2019 - -ENV MINIO_ACCESS_KEY=MinioUser -ENV MINIO_SECRET_KEY=OpenSesame - -RUN curl.exe --output minio.exe --url https://dl.min.io/server/minio/release/windows-amd64/minio.exe - -RUN mkdir c:\data - -CMD minio.exe server --address=:9000 c:\data \ No newline at end of file diff --git a/docker/Windows-JDK15/docker-compose.yml b/docker/Windows-JDK15/docker-compose.yml deleted file mode 100644 index e2c44b801..000000000 --- a/docker/Windows-JDK15/docker-compose.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# N.B.: docker-compose must be invoked from the project root directory: -# -# docker-compose -f path/to/docker-compose.yml up --exit-code-from cantaloupe -# -version: '3' -services: - cantaloupe: - build: - context: ../../ - dockerfile: docker/Windows-JDK15/Dockerfile - minio: - build: - context: ../../ - dockerfile: docker/Windows-JDK15/Dockerfile-minio - environment: - MINIO_ACCESS_KEY: MinioUser - MINIO_SECRET_KEY: OpenSesame - hostname: minio \ No newline at end of file diff --git a/docker/Windows-JDK16/Dockerfile b/docker/Windows-JDK18/Dockerfile similarity index 89% rename from docker/Windows-JDK16/Dockerfile rename to docker/Windows-JDK18/Dockerfile index dc0c9673a..1f4bdd197 100644 --- a/docker/Windows-JDK16/Dockerfile +++ b/docker/Windows-JDK18/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/windows/servercore:ltsc2019 +FROM mcr.microsoft.com/windows/servercore:ltsc2022 ENV chocolateyUseWindowsCompression false @@ -10,7 +10,7 @@ RUN powershell -Command \ # Install various dependencies # TODO: openjpeg -RUN choco install -y adoptopenjdk16 maven ffmpeg +RUN choco install -y adoptopenjdk18 maven ffmpeg # Install TurboJpegProcessor dependencies TODO: libjpeg-turbo #RUN mkdir -p /opt/libjpeg-turbo/lib diff --git a/docker/Windows-JDK16/Dockerfile-minio b/docker/Windows-JDK18/Dockerfile-minio similarity index 81% rename from docker/Windows-JDK16/Dockerfile-minio rename to docker/Windows-JDK18/Dockerfile-minio index b213b8746..115389668 100644 --- a/docker/Windows-JDK16/Dockerfile-minio +++ b/docker/Windows-JDK18/Dockerfile-minio @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/windows/servercore:ltsc2019 +FROM mcr.microsoft.com/windows/servercore:ltsc2022 ENV MINIO_ACCESS_KEY=MinioUser ENV MINIO_SECRET_KEY=OpenSesame diff --git a/docker/Windows-JDK16/docker-compose.yml b/docker/Windows-JDK18/docker-compose.yml similarity index 77% rename from docker/Windows-JDK16/docker-compose.yml rename to docker/Windows-JDK18/docker-compose.yml index 1a505d28b..3c5812953 100644 --- a/docker/Windows-JDK16/docker-compose.yml +++ b/docker/Windows-JDK18/docker-compose.yml @@ -8,11 +8,11 @@ services: cantaloupe: build: context: ../../ - dockerfile: docker/Windows-JDK16/Dockerfile + dockerfile: docker/Windows-JDK18/Dockerfile minio: build: context: ../../ - dockerfile: docker/Windows-JDK16/Dockerfile-minio + dockerfile: docker/Windows-JDK18/Dockerfile-minio environment: MINIO_ACCESS_KEY: MinioUser MINIO_SECRET_KEY: OpenSesame From 83f2a9f1c9bef471844c4d69f479d0745fe6ac4d Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Thu, 12 May 2022 15:52:14 -0500 Subject: [PATCH 040/106] handle() adds the OperationList to the RequestContext before calling OperationList.applyNonEndpointMutations() (fixes #589) --- CHANGES.md | 2 ++ .../library/cantaloupe/resource/ImageRequestHandler.java | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6146b708b..31442cb8c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -37,6 +37,8 @@ ## 5.0.6 * IIIF information endpoints always return JSON in HTTP 4xx responses. +* Fixed a bug whereby the values of the `operations` and `page_count` keys + in the delegate context were not set. * TurboJpegProcessor is able to generate non-JPEG derivative images, which fixes an HTTP 415 error that would occur when trying to do that. * Fixed a crop-offset bug that could occur when using PdfBoxProcessor to diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandler.java b/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandler.java index 57a7b2cf1..31f1cfb6f 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandler.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/ImageRequestHandler.java @@ -398,10 +398,12 @@ public void handle(OutputStream outputStream) throws Exception { try { fullSize = info.getSize(operationList.getPageIndex()); requestContext.setMetadata(info.getMetadata()); - operationList.applyNonEndpointMutations(info, delegateProxy); - operationList.freeze(); requestContext.setOperationList(operationList, fullSize); requestContext.setPageCount(info.getNumPages()); + // This must be done *after* the request context is fully + // populated, as some of the mutations may depend on it. + operationList.applyNonEndpointMutations(info, delegateProxy); + operationList.freeze(); } catch (IllegalArgumentException | IndexOutOfBoundsException e) { throw new IllegalClientArgumentException(e); } From da2ffdb9bfa269a1b847132a19af06b92d46c4e2 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 9 Aug 2023 14:18:10 +0100 Subject: [PATCH 041/106] Updating libraries --- pom.xml | 60 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/pom.xml b/pom.xml index 9c7b9ce25..a01b9f608 100644 --- a/pom.xml +++ b/pom.xml @@ -70,25 +70,25 @@ net.logstash.logback logstash-logback-encoder - 6.1 + 7.4 com.github.ben-manes.caffeine caffeine - 2.5.6 + 3.1.6 com.google.protobuf protobuf-java - 3.5.1 + 4.0.0-rc-2 com.h2database h2 - 1.4.196 + 2.2.220 test @@ -101,19 +101,19 @@ com.microsoft.azure azure-storage - 5.2.0 + 8.6.6 com.squareup.okhttp3 okhttp - 3.13.1 + 5.0.0-alpha.11 com.zaxxer HikariCP - 2.7.2 + 5.0.1 @@ -126,7 +126,7 @@ commons-io commons-io - 2.4 + 2.13.0 edu.illinois.library @@ -137,63 +137,63 @@ io.lettuce lettuce-core - 6.0.1.RELEASE + 6.2.4.RELEASE javax.xml.bind jaxb-api - 2.3.0 + 2.4.0-b180830.0359 it.geosolutions.imageio-ext imageio-ext-tiff - 1.3.2 + 1.4.7 javax.servlet javax.servlet-api - 3.1.0 + 4.0.1 org.apache.commons commons-lang3 - 3.6 + 3.12.0 org.apache.pdfbox pdfbox - 2.0.23 + 2.0.29 org.apache.pdfbox jbig2-imageio - 3.0.3 + 3.0.4 org.junit.jupiter junit-jupiter-engine - 5.4.2 + 5.10.0-RC1 test org.openjdk.jmh jmh-core - 1.19 + 1.36 test org.openjdk.jmh jmh-generator-annprocess - 1.19 + 1.36 test org.apache.tika tika-core - 1.24.1 + 2.8.0 org.bouncycastle bcprov-jdk15on - 1.64 + 1.70 org.codehaus.janino janino - 2.7.8 + 3.1.10 @@ -330,45 +330,45 @@ org.apache.velocity velocity-engine-core - 2.0 + 2.3 org.seleniumhq.selenium htmlunit-driver - 2.21 + 4.10.0 test org.seleniumhq.selenium selenium-api - 2.53.0 + 4.10.0 test org.seleniumhq.selenium selenium-support - 2.53.0 + 4.10.0 test org.slf4j slf4j-api - 1.7.25 + 2.0.7 org.slf4j jcl-over-slf4j - 1.7.25 + 2.0.7 org.apache.maven.plugins maven-assembly-plugin - 3.1.1 + 3.6.0 maven-plugin From 2648718c8b4b22dbe3038f5665298da92761b0d5 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 9 Aug 2023 14:18:29 +0100 Subject: [PATCH 042/106] Preparing for PDFBox v3 update --- .../illinois/library/cantaloupe/processor/PdfBoxProcessor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/PdfBoxProcessor.java b/src/main/java/edu/illinois/library/cantaloupe/processor/PdfBoxProcessor.java index 0197feab7..6de50010c 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/PdfBoxProcessor.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/PdfBoxProcessor.java @@ -142,7 +142,8 @@ public boolean isSeeking() { private void readDocument() throws IOException { if (doc == null) { final Stopwatch watch = new Stopwatch(); - + // For PDF Box v3 this would need to change to a loader: + // https://pdfbox.apache.org/3.0/migration.html#use-loader-to-get-a-pdf-document if (sourceFile != null) { doc = PDDocument.load(sourceFile.toFile(), getMemoryUsageSetting()); From 872d33cce5b2657bdf46d7d1c8fa1a457b8e10aa Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 9 Aug 2023 14:18:48 +0100 Subject: [PATCH 043/106] Updating tika to v2.8.0 See: https://archive.apache.org/dist/tika/2.0.0-BETA/CHANGES-2.0.0-BETA.txt --- .../java/edu/illinois/library/cantaloupe/image/MediaType.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/MediaType.java b/src/main/java/edu/illinois/library/cantaloupe/image/MediaType.java index 1c4b0b367..d5370f3c0 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/image/MediaType.java +++ b/src/main/java/edu/illinois/library/cantaloupe/image/MediaType.java @@ -12,6 +12,7 @@ import org.apache.tika.detect.Detector; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.parser.AutoDetectParser; import java.io.IOException; @@ -99,7 +100,7 @@ public static List detectMediaTypes(Path path) AutoDetectParser parser = new AutoDetectParser(); Detector detector = parser.getDetector(); Metadata md = new Metadata(); - md.add(Metadata.RESOURCE_NAME_KEY, path.toString()); + md.add(TikaCoreProperties.RESOURCE_NAME_KEY, path.toString()); org.apache.tika.mime.MediaType mediaType = detector.detect(is, md); types.add(new MediaType(mediaType.toString())); } From 68a94fff44255771bced6b5136a19cc44df0ed94 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 9 Aug 2023 17:45:49 +0100 Subject: [PATCH 044/106] Rolling back to 1.3.2 which is latest release --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a01b9f608..02e30dca1 100644 --- a/pom.xml +++ b/pom.xml @@ -151,7 +151,8 @@ it.geosolutions.imageio-ext imageio-ext-tiff - 1.4.7 + + 1.3.2 javax.servlet From 9912e6beb2d2cfbe0147c121a6ce54badad702c7 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Thu, 10 Aug 2023 01:00:47 +0100 Subject: [PATCH 045/106] Rolling back seleniumhq --- pom.xml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 02e30dca1..a1edd0592 100644 --- a/pom.xml +++ b/pom.xml @@ -21,6 +21,7 @@ 9.2.17.0 1.2.8 3.0.0-M5 + 2.53.0 @@ -338,19 +339,19 @@ org.seleniumhq.selenium htmlunit-driver - 4.10.0 + ${seleniumhq.version} test org.seleniumhq.selenium selenium-api - 4.10.0 + ${seleniumhq.version} test org.seleniumhq.selenium selenium-support - 4.10.0 + ${seleniumhq.version} test From 0abacd42fc124582f3c0c6fb2acfc79b408a9045 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 30 Aug 2023 13:53:04 +0100 Subject: [PATCH 046/106] Syncing versions of slf4j and logback --- pom.xml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index a1edd0592..754501dcc 100644 --- a/pom.xml +++ b/pom.xml @@ -19,9 +19,10 @@ 2.11.0 11.0.5 9.2.17.0 - 1.2.8 3.0.0-M5 2.53.0 + 1.2.12 + 1.7.36 @@ -358,13 +359,13 @@ org.slf4j slf4j-api - 2.0.7 + ${slf4j.version} org.slf4j jcl-over-slf4j - 2.0.7 + ${slf4j.version} From 4dc81600c38b8bcde17de46c0dd476fb9166a022 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 30 Aug 2023 15:34:40 +0100 Subject: [PATCH 047/106] Updating maven version as 3.8.5 no longer exists at the URL --- docker/Linux-JDK18/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/Linux-JDK18/Dockerfile b/docker/Linux-JDK18/Dockerfile index 947f543e0..c080846f0 100644 --- a/docker/Linux-JDK18/Dockerfile +++ b/docker/Linux-JDK18/Dockerfile @@ -43,10 +43,10 @@ RUN wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6. && tar xfz openjdk-18_linux-x64_bin.tar.gz \ && mv jdk-18 /opt/jdk \ # Install a newer Maven than the one in apt - && wget -q https://dlcdn.apache.org/maven/maven-3/3.8.5/binaries/apache-maven-3.8.5-bin.tar.gz \ - && tar xfz apache-maven-3.8.5-bin.tar.gz \ - && mv apache-maven-3.8.5 /opt/maven \ - && rm apache-maven-3.8.5-bin.tar.gz + && wget -q https://dlcdn.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz \ + && tar xfz apache-maven-3.8.8-bin.tar.gz \ + && mv apache-maven-3.8.8 /opt/maven \ + && rm apache-maven-3.8.8-bin.tar.gz # A non-root user is needed for some FilesystemSourceTest tests to work. ARG user=cantaloupe From ad8a9c6a5f981bdd13a1deb9ade391a039fcb120 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 30 Aug 2023 16:14:55 +0100 Subject: [PATCH 048/106] Trying upgrading windows version --- docker/Windows-JDK11/Dockerfile-minio | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Windows-JDK11/Dockerfile-minio b/docker/Windows-JDK11/Dockerfile-minio index b213b8746..115389668 100644 --- a/docker/Windows-JDK11/Dockerfile-minio +++ b/docker/Windows-JDK11/Dockerfile-minio @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/windows/servercore:ltsc2019 +FROM mcr.microsoft.com/windows/servercore:ltsc2022 ENV MINIO_ACCESS_KEY=MinioUser ENV MINIO_SECRET_KEY=OpenSesame From 4f29982498c366a4a819c8663b37c63bef3304c6 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 30 Aug 2023 16:36:02 +0100 Subject: [PATCH 049/106] Moving choco package --- docker/Windows-JDK18/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/Windows-JDK18/Dockerfile b/docker/Windows-JDK18/Dockerfile index 1f4bdd197..e3f570efc 100644 --- a/docker/Windows-JDK18/Dockerfile +++ b/docker/Windows-JDK18/Dockerfile @@ -10,8 +10,9 @@ RUN powershell -Command \ # Install various dependencies # TODO: openjpeg -RUN choco install -y adoptopenjdk18 maven ffmpeg - +RUN choco install -y maven ffmpeg +RUN choco install -y openjdk --version=18.0.2 + # Install TurboJpegProcessor dependencies TODO: libjpeg-turbo #RUN mkdir -p /opt/libjpeg-turbo/lib #COPY docker/Windows10-JDK11/image_files/libjpeg-turbo/lib64 c:\windows\system32 From 4d1ef3b745149c65afa1e92a66423037a9daf5c6 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Tue, 5 Sep 2023 17:31:13 +0100 Subject: [PATCH 050/106] Moving creating prepared statement to after blob population --- .../library/cantaloupe/cache/JdbcCache.java | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java index f147f0700..53bbc0fc9 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java @@ -56,7 +56,6 @@ private class ImageBlobOutputStream extends CompletableOutputStream { private final OutputStream blobOutputStream; private final OperationList ops; private final Connection connection; - private final PreparedStatement statement; /** * Constructor for writing derivative images. @@ -82,17 +81,26 @@ private class ImageBlobOutputStream extends CompletableOutputStream { final Blob blob = connection.createBlob(); blobOutputStream = blob.setBinaryStream(1); - statement = connection.prepareStatement(sql); - statement.setString(1, ops.toString()); - statement.setBlob(2, blob); - statement.setTimestamp(3, now()); } @Override public void close() throws IOException { LOGGER.debug("Closing stream for {}", ops); + PreparedStatement statement = null; try { - if (isComplete()) { + if (isCompletelyWritten()) { + final Configuration config = Configuration.getInstance(); + final String sql = String.format( + "INSERT INTO %s (%s, %s, %s) VALUES (?, ?, ?)", + config.getString(Key.JDBCCACHE_DERIVATIVE_IMAGE_TABLE), + DERIVATIVE_IMAGE_TABLE_OPERATIONS_COLUMN, + DERIVATIVE_IMAGE_TABLE_IMAGE_COLUMN, + DERIVATIVE_IMAGE_TABLE_LAST_ACCESSED_COLUMN); + LOGGER.debug(sql); + statement = connection.prepareStatement(sql); + statement.setString(1, ops.toString()); + statement.setBlob(2, blob); + statement.setTimestamp(3, now()); statement.executeUpdate(); connection.commit(); } else { @@ -102,7 +110,9 @@ public void close() throws IOException { throw new IOException(e.getMessage(), e); } finally { try { - statement.close(); + if (statement != null) { + statement.close(); + } } catch (SQLException e) { LOGGER.error(e.getMessage(), e); } @@ -390,6 +400,7 @@ public InputStream newDerivativeImageInputStream(OperationList opList) try { return new ImageBlobOutputStream(getConnection(), ops); } catch (SQLException e) { + LOGGER.error("Throwing Except: {}", e); throw new IOException(e.getMessage(), e); } } From bcad4e30640bb7783e76dec54e2ee408f4458042 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 6 Sep 2023 11:45:23 +0100 Subject: [PATCH 051/106] Fixing a missing blob exception --- .../edu/illinois/library/cantaloupe/cache/JdbcCache.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java index 53bbc0fc9..4d882deb1 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java @@ -56,6 +56,7 @@ private class ImageBlobOutputStream extends CompletableOutputStream { private final OutputStream blobOutputStream; private final OperationList ops; private final Connection connection; + private final Blob blob; /** * Constructor for writing derivative images. @@ -80,6 +81,7 @@ private class ImageBlobOutputStream extends CompletableOutputStream { LOGGER.trace(sql); final Blob blob = connection.createBlob(); + blob = connection.createBlob(); blobOutputStream = blob.setBinaryStream(1); } @@ -89,6 +91,7 @@ public void close() throws IOException { PreparedStatement statement = null; try { if (isCompletelyWritten()) { + blobOutputStream.close(); final Configuration config = Configuration.getInstance(); final String sql = String.format( "INSERT INTO %s (%s, %s, %s) VALUES (?, ?, ?)", @@ -371,7 +374,7 @@ public InputStream newDerivativeImageInputStream(OperationList opList) DERIVATIVE_IMAGE_TABLE_LAST_ACCESSED_COLUMN); try (Connection conn = getConnection(); - PreparedStatement statement = conn.prepareStatement(sql)) { + PreparedStatement statement = conn.prepareStatement(sql)) { statement.setString(1, opList.toString()); statement.setTimestamp(2, earliestValidDate()); From 7884ea70c46a403d570bb7337cb571cf0f9eff6d Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 6 Sep 2023 11:46:20 +0100 Subject: [PATCH 052/106] Rolling back HikariCP to last version to support Java 8 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 754501dcc..7fd166a4b 100644 --- a/pom.xml +++ b/pom.xml @@ -115,7 +115,7 @@ com.zaxxer HikariCP - 5.0.1 + 4.0.3 From 9c5a6070e3d91369f8c67e382d2a7f97a942e673 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Tue, 3 Oct 2023 15:28:57 +0100 Subject: [PATCH 053/106] Adding a baseURI to make the rdf:about have a value --- .../cantaloupe/processor/codec/tiff/TIFFMetadataTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/illinois/library/cantaloupe/processor/codec/tiff/TIFFMetadataTest.java b/src/test/java/edu/illinois/library/cantaloupe/processor/codec/tiff/TIFFMetadataTest.java index 7104e06be..132cf7bc9 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/processor/codec/tiff/TIFFMetadataTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/processor/codec/tiff/TIFFMetadataTest.java @@ -4,6 +4,8 @@ import edu.illinois.library.cantaloupe.test.TestUtil; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.riot.RDFDataMgr; +import org.apache.jena.riot.RDFFormat; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -80,7 +82,7 @@ void testGetXMP() throws IOException { try (ImageInputStream is = ImageIO.createImageInputStream(srcFile.toFile())) { final String rdf = newInstance(is).getXMP().orElseThrow(); final Model model = ModelFactory.createDefaultModel(); - model.read(new StringReader(rdf), null, "RDF/XML"); + model.read(new StringReader(rdf), "file://" + srcFile.getParent().toAbsolutePath(), "RDF/XML"); } } From f848f6faa2c9af80dd4419d50682a060fb3c1c5a Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Tue, 3 Oct 2023 15:29:32 +0100 Subject: [PATCH 054/106] Adding an rdf:about value if not present --- .../library/cantaloupe/processor/codec/jpeg/Util.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/Util.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/Util.java index b1fce36b0..8e50506c1 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/Util.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/Util.java @@ -119,8 +119,13 @@ private static String mergeXMPModels(String standardXMP, private static Model readModel(String rdfXML) { Model model = ModelFactory.createDefaultModel(); + String base = null; + if (rdfXML.indexOf("rdf:about=''") != -1 || rdfXML.indexOf("rdf:about=\"\"") != -1) { + // Verison 4.8+ of jena requires a rdf:about link to not be empty + base = "http://example.com"; + } try (StringReader reader = new StringReader(rdfXML)) { - model.read(reader, null, "RDF/XML"); + model.read(reader, base, "RDF/XML"); } return model; } From 4af866c1d2215d3d0ef0ff6622fd980d64bdb729 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Tue, 3 Oct 2023 15:30:02 +0100 Subject: [PATCH 055/106] Updating to latest Jena version --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 7fd166a4b..35ea1c2fb 100644 --- a/pom.xml +++ b/pom.xml @@ -205,12 +205,12 @@ org.apache.jena jena-core - 4.8.0 + 4.9.0 org.apache.jena jena-arq - 4.8.0 + 4.9.0 From 40c3412496b565ef8ab705ae8bdf3e7780990d35 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Tue, 3 Oct 2023 16:21:59 +0100 Subject: [PATCH 056/106] Checking for empty rdf:about and using base URI --- .../library/cantaloupe/image/Metadata.java | 21 +++++++++++++++++-- .../cantaloupe/processor/codec/jpeg/Util.java | 14 +++++++++---- .../cantaloupe/image/MetadataTest.java | 2 ++ .../processor/codec/png/PNGMetadataTest.java | 5 +++-- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java b/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java index 1baa942c2..cb3e29095 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java +++ b/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java @@ -209,10 +209,27 @@ private void loadXMP() { RIOT.init(); xmpModel = ModelFactory.createDefaultModel(); + String base = null; + if (xmp.get().indexOf("rdf:about=''") != -1 || xmp.get().indexOf("rdf:about=\"\"") != -1) { + // Version 4.8+ of jena requires a rdf:about link to not be empty + base = "http://example.com"; + } try (StringReader reader = new StringReader(xmp.get())) { - xmpModel.read(reader, null, "RDF/XML"); - } catch (RiotException | NullPointerException e) { + xmpModel.read(reader, base, "RDF/XML"); + } catch (RiotException e) { + if (e.getMessage().indexOf("Base URI is null, but there are relative URIs to resolve") != -1) { + // Version 4.8+ of jena requires a rdf:about link to not be empty + try (StringReader reader = new StringReader(xmp.get())) { + xmpModel.read(reader, "http://example.com", "RDF/XML"); + } catch (RiotException exception) { + LOGGER.info("loadXMP(): {}", exception.getMessage()); + } + } else { + LOGGER.info("loadXMP(): {}", e.getMessage()); + throw e; + } + } catch (NullPointerException e) { // The XMP string may be invalid RDF/XML, or there may be a bug // in Jena (that would be the NPE). Not much we can do. LOGGER.info("loadXMP(): {}", e.getMessage()); diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/Util.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/Util.java index 8e50506c1..c73507dd4 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/Util.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/Util.java @@ -6,6 +6,7 @@ import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.rdf.model.RDFNode; import org.apache.jena.rdf.model.StmtIterator; +import org.apache.jena.riot.RiotException; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -120,12 +121,17 @@ private static String mergeXMPModels(String standardXMP, private static Model readModel(String rdfXML) { Model model = ModelFactory.createDefaultModel(); String base = null; - if (rdfXML.indexOf("rdf:about=''") != -1 || rdfXML.indexOf("rdf:about=\"\"") != -1) { - // Verison 4.8+ of jena requires a rdf:about link to not be empty - base = "http://example.com"; - } try (StringReader reader = new StringReader(rdfXML)) { model.read(reader, base, "RDF/XML"); + } catch (RiotException exception) { + if (exception.getMessage().indexOf("Base URI is null, but there are relative URIs to resolve") != -1) { + // Version 4.8+ of jena requires a rdf:about link to not be empty + try (StringReader reader = new StringReader(rdfXML)) { + model.read(reader, "http://example.com", "RDF/XML"); + } + } else { + throw exception; + } } return model; } diff --git a/src/test/java/edu/illinois/library/cantaloupe/image/MetadataTest.java b/src/test/java/edu/illinois/library/cantaloupe/image/MetadataTest.java index 366c9cb77..10a2846ff 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/image/MetadataTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/image/MetadataTest.java @@ -12,6 +12,8 @@ import edu.illinois.library.cantaloupe.test.BaseTest; import edu.illinois.library.cantaloupe.test.TestUtil; import org.apache.jena.rdf.model.Model; +import org.apache.jena.riot.RDFDataMgr; +import org.apache.jena.riot.RDFFormat; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/src/test/java/edu/illinois/library/cantaloupe/processor/codec/png/PNGMetadataTest.java b/src/test/java/edu/illinois/library/cantaloupe/processor/codec/png/PNGMetadataTest.java index cf709ab1a..31ddace80 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/processor/codec/png/PNGMetadataTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/processor/codec/png/PNGMetadataTest.java @@ -45,9 +45,10 @@ void testGetNativeMetadata() throws IOException { @Test void testGetXMP() throws IOException { - final String rdf = getInstance("png-xmp.png").getXMP().orElseThrow(); + final String fixtureName = "png-xmp.png"; + final String rdf = getInstance(fixtureName).getXMP().orElseThrow(); final Model model = ModelFactory.createDefaultModel(); - model.read(new StringReader(rdf), null, "RDF/XML"); + model.read(new StringReader(rdf), "file://" + TestUtil.getImage(fixtureName).getParent().toAbsolutePath(), "RDF/XML"); } } From c961380fee2343ae92cba0ec45e95b08490a58ee Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Tue, 3 Oct 2023 16:35:50 +0100 Subject: [PATCH 057/106] Unpinning htmlunit version form selenium --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 35ea1c2fb..c361862a2 100644 --- a/pom.xml +++ b/pom.xml @@ -340,7 +340,7 @@ org.seleniumhq.selenium htmlunit-driver - ${seleniumhq.version} + 2.21 test From 7be32ce88bd5788404e2a5bf90d0bd5d75c4ee97 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 4 Oct 2023 11:29:29 +0100 Subject: [PATCH 058/106] Handling different exception but same outcome --- .../java/edu/illinois/library/cantaloupe/image/Metadata.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java b/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java index cb3e29095..8a2d26819 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java +++ b/src/main/java/edu/illinois/library/cantaloupe/image/Metadata.java @@ -123,6 +123,9 @@ public Orientation getOrientation() { } catch (IllegalArgumentException e) { LOGGER.info("readOrientation(): {}", e.getMessage()); orientation = Orientation.ROTATE_0; + } catch (RiotException e) { + LOGGER.info("readOrientation(): {}", e.getMessage()); + orientation = Orientation.ROTATE_0; } } return orientation; From 72cad8ba908621220aac79e8c72e0db38148d53b Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 4 Oct 2023 11:29:54 +0100 Subject: [PATCH 059/106] Adding RDF namespace --- .../edu/illinois/library/cantaloupe/image/MetadataTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/illinois/library/cantaloupe/image/MetadataTest.java b/src/test/java/edu/illinois/library/cantaloupe/image/MetadataTest.java index 10a2846ff..de3d914f6 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/image/MetadataTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/image/MetadataTest.java @@ -446,7 +446,7 @@ void testToMap() { expectedMap.put("iptc", List.of(new DataSet( edu.illinois.library.cantaloupe.image.iptc.Tag.CITY, "Urbana".getBytes()).toMap())); - expectedMap.put("xmp_string", ""); + expectedMap.put("xmp_string", ""); expectedMap.put("xmp_elements", Collections.emptyMap()); expectedMap.put("native", Map.of("key1", "value1", "key2", "value2")); @@ -466,7 +466,7 @@ void testToMap() { "Urbana".getBytes())); instance.setIPTC(iptc); // XMP - instance.setXMP(""); + instance.setXMP(""); // native instance.setNativeMetadata(Map.of("key1", "value1", "key2", "value2")); From 1d18c9320d823cbf44c5328b985f65297ce352b9 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 4 Oct 2023 11:32:04 +0100 Subject: [PATCH 060/106] Adding RDF namespace to test --- .../edu/illinois/library/cantaloupe/operation/EncodeTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/illinois/library/cantaloupe/operation/EncodeTest.java b/src/test/java/edu/illinois/library/cantaloupe/operation/EncodeTest.java index ee8f5b614..0fc0b9aac 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/operation/EncodeTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/operation/EncodeTest.java @@ -137,7 +137,7 @@ void testToMap() { instance.setBackgroundColor(Color.BLUE); instance.setMaxComponentSize(10); Metadata metadata = new Metadata(); - metadata.setXMP(""); + metadata.setXMP(""); instance.setMetadata(metadata); Dimension size = new Dimension(500, 500); @@ -150,7 +150,7 @@ void testToMap() { assertTrue((boolean) map.get("interlace")); assertEquals(50, map.get("quality")); assertEquals(10, map.get("max_sample_size")); - assertEquals("", + assertEquals("", ((Map) map.get("metadata")).get("xmp_string")); } From 989cf93bd96cda9d373af238e8e424f31b3708d3 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 4 Oct 2023 11:55:47 +0100 Subject: [PATCH 061/106] Updating Jackson --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c361862a2..ce6a5b9c8 100644 --- a/pom.xml +++ b/pom.xml @@ -16,8 +16,8 @@ UTF-8 UTF-8 2.15.28 - 2.11.0 11.0.5 + 2.15.2 9.2.17.0 3.0.0-M5 2.53.0 @@ -572,6 +572,7 @@ RedisCacheTest TurboJpegProcessorTest TurboJPEG*Test + *S3* From 7cbae2c3042c8a0e70432debf4dbd30e8b95b779 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 4 Oct 2023 12:52:34 +0100 Subject: [PATCH 062/106] Updating jetty --- pom.xml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index ce6a5b9c8..c3bca3765 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 2.15.28 11.0.5 2.15.2 - 9.2.17.0 + 9.4.3.0 3.0.0-M5 2.53.0 1.2.12 @@ -139,7 +139,7 @@ io.lettuce lettuce-core - 6.2.4.RELEASE + 6.2.6.RELEASE @@ -221,8 +221,8 @@ org.bouncycastle - bcprov-jdk15on - 1.70 + bcprov-jdk18on + 1.76 @@ -503,7 +503,7 @@ maven-assembly-plugin - 3.1.1 + 3.6.0 package @@ -572,7 +572,6 @@ RedisCacheTest TurboJpegProcessorTest TurboJPEG*Test - *S3* From 82c2b6ffef31d2e37bcc1e6e8b7995b5be225be7 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Tue, 5 Dec 2023 15:33:49 +0000 Subject: [PATCH 063/106] Moving to CGI.escape for Ruby 3 --- src/test/resources/delegates.rb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/test/resources/delegates.rb b/src/test/resources/delegates.rb index 3ad145081..509ee0dda 100644 --- a/src/test/resources/delegates.rb +++ b/src/test/resources/delegates.rb @@ -2,6 +2,7 @@ require 'java' require 'uri' +require 'cgi' class CustomDelegate @@ -166,11 +167,11 @@ def httpsource_resource_info(options = {}) elsif context['client_ip'] == '1.2.3.4' if context['request_headers']['X-Forwarded-Proto'] == 'https' return { - 'uri' => 'https://other-example.org/bleh/' + URI.escape(identifier) + 'uri' => 'https://other-example.org/bleh/' + CGI.escape(identifier) } else return { - 'uri' => 'http://other-example.org/bleh/' + URI.escape(identifier) + 'uri' => 'http://other-example.org/bleh/' + CGI.escape(identifier) } end end @@ -178,7 +179,7 @@ def httpsource_resource_info(options = {}) case identifier when 'http-jpg-rgb-64x56x8-baseline.jpg' return { - 'uri' => 'http://example.org/bla/' + URI.escape(identifier), + 'uri' => 'http://example.org/bla/' + CGI.escape(identifier), 'headers' => { 'X-Custom' => 'yes' }, @@ -186,7 +187,7 @@ def httpsource_resource_info(options = {}) } when 'https-jpg-rgb-64x56x8-baseline.jpg' return { - 'uri' => 'https://example.org/bla/' + URI.escape(identifier), + 'uri' => 'https://example.org/bla/' + CGI.escape(identifier), 'headers' => { 'X-Custom' => 'yes' }, @@ -194,7 +195,7 @@ def httpsource_resource_info(options = {}) } when 'http-jpg-rgb-64x56x8-plane.jpg' return { - 'uri' => 'http://example.org/bla/' + URI.escape(identifier), + 'uri' => 'http://example.org/bla/' + CGI.escape(identifier), 'username' => 'username', 'secret' => 'secret', 'headers' => { @@ -204,7 +205,7 @@ def httpsource_resource_info(options = {}) } when 'https-jpg-rgb-64x56x8-plane.jpg' return { - 'uri' => 'https://example.org/bla/' + URI.escape(identifier), + 'uri' => 'https://example.org/bla/' + CGI.escape(identifier), 'username' => 'username', 'secret' => 'secret', 'headers' => { From 8b80659910429df454a0f7ecccf4f660c8b0601a Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Tue, 5 Dec 2023 16:27:08 +0000 Subject: [PATCH 064/106] Trying the debian version of grok --- docker/Linux-JDK11/Dockerfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docker/Linux-JDK11/Dockerfile b/docker/Linux-JDK11/Dockerfile index fb3eab9d5..c491f5a5e 100644 --- a/docker/Linux-JDK11/Dockerfile +++ b/docker/Linux-JDK11/Dockerfile @@ -18,6 +18,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ zlib1g-dev \ libwebp-dev \ libimage-exiftool-perl \ + libgrokj2k1 \ + grokj2k-tools && rm -rf /var/lib/apt/lists/* # Install TurboJpegProcessor dependencies @@ -28,10 +30,10 @@ COPY docker/Linux-JDK11/image_files/libjpeg-turbo/lib64 /opt/libjpeg-turbo/lib COPY dist/deps/Linux-x86-64/lib/* /usr/lib/ # Install GrokProcessor dependencies -RUN wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/libgrokj2k1_7.6.5-1_amd64.deb \ - && wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/grokj2k-tools_7.6.5-1_amd64.deb \ - && dpkg -i ./libgrokj2k1_7.6.5-1_amd64.deb \ - && dpkg -i --ignore-depends=libjpeg62-turbo ./grokj2k-tools_7.6.5-1_amd64.deb +#RUN wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/libgrokj2k1_7.6.5-1_amd64.deb \ +# && wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/grokj2k-tools_7.6.5-1_amd64.deb \ +# && dpkg -i ./libgrokj2k1_7.6.5-1_amd64.deb \ +# && dpkg -i --ignore-depends=libjpeg62-turbo ./grokj2k-tools_7.6.5-1_amd64.deb # A non-root user is needed for some FilesystemSourceTest tests to work. ARG user=cantaloupe From 3f784025f02b2314253f4ba9ec6795d3203943bb Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Tue, 5 Dec 2023 16:28:22 +0000 Subject: [PATCH 065/106] Fixing typo --- docker/Linux-JDK11/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Linux-JDK11/Dockerfile b/docker/Linux-JDK11/Dockerfile index c491f5a5e..6070879f1 100644 --- a/docker/Linux-JDK11/Dockerfile +++ b/docker/Linux-JDK11/Dockerfile @@ -19,7 +19,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libwebp-dev \ libimage-exiftool-perl \ libgrokj2k1 \ - grokj2k-tools + grokj2k-tools \ && rm -rf /var/lib/apt/lists/* # Install TurboJpegProcessor dependencies From 42c3e2a082febab6402d33cf34d2868428c6a0fc Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Tue, 5 Dec 2023 16:36:52 +0000 Subject: [PATCH 066/106] Updating Ubuntu to version with grok --- docker/Linux-JDK11/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Linux-JDK11/Dockerfile b/docker/Linux-JDK11/Dockerfile index 6070879f1..52cc01447 100644 --- a/docker/Linux-JDK11/Dockerfile +++ b/docker/Linux-JDK11/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:latest +FROM ubuntu:lunar ARG DEBIAN_FRONTEND=noninteractive From bb4117344fe203a79660ce52382cb67a8663f0b7 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 6 Dec 2023 23:59:34 +0000 Subject: [PATCH 067/106] Moving to ubuntu lunar so we can get Grok via apt-get --- docker/Linux-JDK11/Dockerfile | 3 ++- docker/Linux-JDK18/Dockerfile | 15 +++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docker/Linux-JDK11/Dockerfile b/docker/Linux-JDK11/Dockerfile index 52cc01447..3129f9a73 100644 --- a/docker/Linux-JDK11/Dockerfile +++ b/docker/Linux-JDK11/Dockerfile @@ -20,6 +20,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libimage-exiftool-perl \ libgrokj2k1 \ grokj2k-tools \ + adduser \ && rm -rf /var/lib/apt/lists/* # Install TurboJpegProcessor dependencies @@ -39,7 +40,7 @@ COPY dist/deps/Linux-x86-64/lib/* /usr/lib/ ARG user=cantaloupe ARG home=/home/$user RUN adduser --home $home $user -RUN chown -R $user $home +RUN chown -R $user $home USER $user WORKDIR $home diff --git a/docker/Linux-JDK18/Dockerfile b/docker/Linux-JDK18/Dockerfile index c080846f0..ac131292b 100644 --- a/docker/Linux-JDK18/Dockerfile +++ b/docker/Linux-JDK18/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:latest +FROM ubuntu:lunar ENV JAVA_HOME=/opt/jdk ENV PATH=$PATH:/opt/jdk/bin:/opt/maven/bin @@ -23,6 +23,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ zlib1g-dev \ libwebp-dev \ libimage-exiftool-perl \ + libgrokj2k1 \ + grokj2k-tools \ + adduser \ && rm -rf /var/lib/apt/lists/* # Install TurboJpegProcessor dependencies @@ -34,12 +37,12 @@ COPY dist/deps/Linux-x86-64/lib/* /usr/lib/ # Install various other dependencies that aren't in apt # Install GrokProcessor dependencies -RUN wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/libgrokj2k1_7.6.5-1_amd64.deb \ - && wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/grokj2k-tools_7.6.5-1_amd64.deb \ - && dpkg -i ./libgrokj2k1_7.6.5-1_amd64.deb \ - && dpkg -i --ignore-depends=libjpeg62-turbo ./grokj2k-tools_7.6.5-1_amd64.deb \ +#RUN wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/libgrokj2k1_7.6.5-1_amd64.deb \ +# && wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/grokj2k-tools_7.6.5-1_amd64.deb \ +# && dpkg -i ./libgrokj2k1_7.6.5-1_amd64.deb \ +# && dpkg -i --ignore-depends=libjpeg62-turbo ./grokj2k-tools_7.6.5-1_amd64.deb \ # Install OpenJDK - && wget -q https://download.java.net/java/GA/jdk18/43f95e8614114aeaa8e8a5fcf20a682d/36/GPL/openjdk-18_linux-x64_bin.tar.gz \ +RUN wget -q https://download.java.net/java/GA/jdk18/43f95e8614114aeaa8e8a5fcf20a682d/36/GPL/openjdk-18_linux-x64_bin.tar.gz \ && tar xfz openjdk-18_linux-x64_bin.tar.gz \ && mv jdk-18 /opt/jdk \ # Install a newer Maven than the one in apt From dc1314b0e141468fe337a079ced782c5f5e05c4c Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Thu, 7 Dec 2023 00:22:32 +0000 Subject: [PATCH 068/106] Updating grok install --- docker/Linux-GraalVM20/Dockerfile | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docker/Linux-GraalVM20/Dockerfile b/docker/Linux-GraalVM20/Dockerfile index 847a3795f..55b7c044c 100644 --- a/docker/Linux-GraalVM20/Dockerfile +++ b/docker/Linux-GraalVM20/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:latest +FROM ubuntu:lunar ENV JAVA_HOME=/opt/graalvm-ce-java11-20.3.0 ENV GRAALVM_HOME=/opt/graalvm-ce-java11-20.3.0 @@ -20,6 +20,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ zlib1g-dev \ libwebp-dev \ libimage-exiftool-perl \ + libgrokj2k1 \ + grokj2k-tools \ + adduser \ && rm -rf /var/lib/apt/lists/* # Install TurboJpegProcessor dependencies @@ -30,10 +33,10 @@ COPY docker/Linux-JDK11/image_files/libjpeg-turbo/lib64 /opt/libjpeg-turbo/lib COPY dist/deps/Linux-x86-64/lib/* /usr/lib/ # Install GrokProcessor dependencies -RUN wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/libgrokj2k1_7.6.5-1_amd64.deb \ - && wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/grokj2k-tools_7.6.5-1_amd64.deb \ - && dpkg -i ./libgrokj2k1_7.6.5-1_amd64.deb \ - && dpkg -i --ignore-depends=libjpeg62-turbo ./grokj2k-tools_7.6.5-1_amd64.deb +#RUN wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/libgrokj2k1_7.6.5-1_amd64.deb \ +# && wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/grokj2k-tools_7.6.5-1_amd64.deb \ +# && dpkg -i --ignore-depends=libjpeg62-turbo ./grokj2k-tools_7.6.5-1_amd64.deb +# && dpkg -i ./libgrokj2k1_7.6.5-1_amd64.deb \ # Install GraalVM RUN wget -q https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-20.3.0/graalvm-ce-java11-linux-amd64-20.3.0.tar.gz \ From 9f12ed3e3a5d5d2934c686b3646691b0d21b7ca8 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 24 Jan 2024 12:47:30 +0000 Subject: [PATCH 069/106] Adding changes to v 5.0.6 --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 31442cb8c..e3ac2baaa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -43,6 +43,8 @@ fixes an HTTP 415 error that would occur when trying to do that. * Fixed a crop-offset bug that could occur when using PdfBoxProcessor to generate JPEGs with libjpeg-turbo active. +* Updating libraries to fix security issues. Full details in [#634](https://github.com/cantaloupe-project/cantaloupe/issues/634) +* Update of Jena to 4.8 requires RDF to have a populated rdf:about field. May impact some XMP header processing. ## 5.0.5 From b9f0cd9315710e01e369132a32c936bf54810954 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 24 Jan 2024 12:59:56 +0000 Subject: [PATCH 070/106] Adding basic CREDITS.md --- CREDITS.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 CREDITS.md diff --git a/CREDITS.md b/CREDITS.md new file mode 100644 index 000000000..e0849d234 --- /dev/null +++ b/CREDITS.md @@ -0,0 +1,9 @@ +# Credits + +The success of this project is down to the great work done by the developers at the University of Illinois and particularly Alex Dolski who has been the main developer on the project. In 2023 the IIIF consortium offered to help release a security patch as so many IIIF community members are using this great software. + +- Cantaloupe 5.0.6 + * [Glen Robson](https://github.com/glenrobson/) with support from the [IIIF Consortium](https://iiif.io/community/consortium/) + +- Cantaloupe v1 - v6 + * [Alex Dolski](https://github.com/adolski), University of Illinois \ No newline at end of file From 396a7cdb4264d1d691372c768a6f1c85ff171537 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 24 Jan 2024 14:43:52 +0000 Subject: [PATCH 071/106] Updating logback version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c3bca3765..9d1d163c7 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ 9.4.3.0 3.0.0-M5 2.53.0 - 1.2.12 + 1.2.13 1.7.36 From f0f1349ac1db0333db69e1be6a5d6713600b3768 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 24 Jan 2024 15:01:51 +0000 Subject: [PATCH 072/106] Updating libraries for security --- pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 9d1d163c7..25ee26a02 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ 2.15.2 9.4.3.0 3.0.0-M5 - 2.53.0 + 2.53.1 1.2.13 1.7.36 @@ -109,7 +109,7 @@ com.squareup.okhttp3 okhttp - 5.0.0-alpha.11 + 5.0.0-alpha.12 @@ -205,12 +205,12 @@ org.apache.jena jena-core - 4.9.0 + 4.10.0 org.apache.jena jena-arq - 4.9.0 + 4.10.0 From 30d6fcf3fa98fd320aac88578ca9444fce3b744d Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 24 Jan 2024 15:40:57 +0000 Subject: [PATCH 073/106] Updating mvn verify plugin --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 25ee26a02..1acff3219 100644 --- a/pom.xml +++ b/pom.xml @@ -534,7 +534,7 @@ org.owasp dependency-check-maven - 5.2.4 + 9.0.9 From b65750fc9550062e23237e44bdd0e9d96ec5f596 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 24 Jan 2024 16:18:50 +0000 Subject: [PATCH 074/106] Updating libraries identified by mvn verify --- pom.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 1acff3219..aecdbe29d 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,6 @@ 11 UTF-8 UTF-8 - 2.15.28 11.0.5 2.15.2 9.4.3.0 @@ -139,7 +138,7 @@ io.lettuce lettuce-core - 6.2.6.RELEASE + 6.2.7.RELEASE From e4e08611dbbd0f569b123dff4ce4d5cc67e16353 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 24 Jan 2024 16:39:59 +0000 Subject: [PATCH 075/106] Updating spot bugs --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index aecdbe29d..17231efaa 100644 --- a/pom.xml +++ b/pom.xml @@ -432,13 +432,13 @@ com.github.spotbugs spotbugs-maven-plugin - 4.1.3 + 4.8.3.0 com.github.spotbugs spotbugs - 4.1.4 + 4.8.3 From 2aad52d12da929d308e5b3cd282e98c20af45564 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 24 Jan 2024 16:53:14 +0000 Subject: [PATCH 076/106] Rolling back s3 version --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 17231efaa..47fdc0b0c 100644 --- a/pom.xml +++ b/pom.xml @@ -16,6 +16,7 @@ UTF-8 UTF-8 11.0.5 + 2.15.28 2.15.2 9.4.3.0 3.0.0-M5 From 634b6e01cc2ca3c84633b69f752e6f706d46c925 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 24 Jan 2024 18:28:38 +0000 Subject: [PATCH 077/106] Adding wait before testing if purged --- .../illinois/library/cantaloupe/cache/AbstractCacheTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java index f2260977b..7a8927ff5 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java @@ -325,6 +325,9 @@ void testPurge() throws Exception { // purge everything instance.purge(); + // Allow time for purge + Thread.sleep(ASYNC_WAIT); + // assert that the info has been purged assertFalse(instance.getInfo(identifier).isPresent()); From f49672cc28361d8103a36aab5c0488f8f0a9cbc8 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 24 Jan 2024 18:47:45 +0000 Subject: [PATCH 078/106] Speeding up test --- .../illinois/library/cantaloupe/cache/AbstractCacheTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java index 7a8927ff5..8b04f03d6 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/AbstractCacheTest.java @@ -325,8 +325,8 @@ void testPurge() throws Exception { // purge everything instance.purge(); - // Allow time for purge - Thread.sleep(ASYNC_WAIT); + // Allow time for purge but not as long as upload + Thread.sleep(ASYNC_WAIT / 2); // assert that the info has been purged assertFalse(instance.getInfo(identifier).isPresent()); From fb4fe2c5ba91cdb97a3bdd1cb849b62b88b9a999 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 24 Jan 2024 19:05:34 +0000 Subject: [PATCH 079/106] Adding a pause after purge --- .../edu/illinois/library/cantaloupe/cache/S3CacheTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java index 1d534c6c2..6f00ec903 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java @@ -317,6 +317,9 @@ void testPurgeWithKeyPrefix() throws Exception { // purge everything instance.purge(); + // Allow some time for the purge to succeed + Thread.sleep(ASYNC_WAIT / 2); + // assert that the info has been purged assertFalse(instance.getInfo(identifier).isPresent()); From 12e9529e7b410f53fb69d7a0c639dc8ede01c3dc Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 24 Jan 2024 19:05:51 +0000 Subject: [PATCH 080/106] Trying a newer version of AWS --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 47fdc0b0c..6141292e1 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,7 @@ UTF-8 UTF-8 11.0.5 - 2.15.28 + 2.21.4 2.15.2 9.4.3.0 3.0.0-M5 From a0ed514be2040de294d2c56a202c282932dc0acb Mon Sep 17 00:00:00 2001 From: angelahuqing Date: Wed, 8 May 2024 14:50:39 -0700 Subject: [PATCH 081/106] Replace Jakarta library with Javax --- .../cantaloupe/resource/AbstractResource.java | 4 ++-- .../library/cantaloupe/resource/FileServlet.java | 6 +++--- .../library/cantaloupe/resource/HandlerServlet.java | 6 +++--- .../library/cantaloupe/resource/Request.java | 2 +- .../illinois/library/cantaloupe/http/ServerTest.java | 4 ++-- .../cantaloupe/resource/MockHttpServletRequest.java | 12 ++++++------ .../cantaloupe/resource/MockHttpServletResponse.java | 4 ++-- .../cantaloupe/source/HTTPStreamFactoryTest.java | 4 ++-- .../library/cantaloupe/source/HttpSourceTest.java | 4 ++-- .../source/OkHttpHTTPImageInputStreamClientTest.java | 4 ++-- 10 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/AbstractResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/AbstractResource.java index 7b2c04e8d..20181b298 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/AbstractResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/AbstractResource.java @@ -21,7 +21,7 @@ import edu.illinois.library.cantaloupe.util.StringUtils; import org.slf4j.Logger; -import jakarta.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; @@ -41,7 +41,7 @@ * more of the HTTP-method-specific methods {@link #doGET()} etc., and may * optionally use {@link #doInit()} and {@link #destroy()}.

      * - *

      Unlike {@link jakarta.servlet.http.HttpServlet}s, instances are only used + *

      Unlike {@link javax.servlet.http.HttpServlet}s, instances are only used * once and not shared across threads.

      */ public abstract class AbstractResource { diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/FileServlet.java b/src/main/java/edu/illinois/library/cantaloupe/resource/FileServlet.java index 94551c415..cbad65f02 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/FileServlet.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/FileServlet.java @@ -1,8 +1,8 @@ package edu.illinois.library.cantaloupe.resource; -import jakarta.servlet.http.HttpServlet; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/HandlerServlet.java b/src/main/java/edu/illinois/library/cantaloupe/resource/HandlerServlet.java index b84025a3d..2223870ec 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/HandlerServlet.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/HandlerServlet.java @@ -8,9 +8,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import jakarta.servlet.http.HttpServlet; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.PrintWriter; import java.util.List; diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/Request.java b/src/main/java/edu/illinois/library/cantaloupe/resource/Request.java index 9e36c9a38..55588eab4 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/Request.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/Request.java @@ -6,7 +6,7 @@ import edu.illinois.library.cantaloupe.http.Query; import edu.illinois.library.cantaloupe.http.Reference; -import jakarta.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletRequest; import java.io.IOException; import java.io.InputStream; import java.util.Enumeration; diff --git a/src/test/java/edu/illinois/library/cantaloupe/http/ServerTest.java b/src/test/java/edu/illinois/library/cantaloupe/http/ServerTest.java index 1e6002d58..84c79a215 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/http/ServerTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/http/ServerTest.java @@ -9,8 +9,8 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import static org.junit.jupiter.api.Assertions.*; diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletRequest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletRequest.java index 1bdb796ab..3b72c90f5 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletRequest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletRequest.java @@ -7,12 +7,12 @@ import jakarta.servlet.ServletInputStream; import jakarta.servlet.ServletRequest; import jakarta.servlet.ServletResponse; -import jakarta.servlet.http.Cookie; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; -import jakarta.servlet.http.HttpSession; -import jakarta.servlet.http.HttpUpgradeHandler; -import jakarta.servlet.http.Part; +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpSession; +import javax.servlet.http.HttpUpgradeHandler; +import javax.servlet.http.Part; import java.io.BufferedReader; import java.security.Principal; import java.util.Collection; diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletResponse.java b/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletResponse.java index 638cd4599..749e314f3 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletResponse.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletResponse.java @@ -3,8 +3,8 @@ import edu.illinois.library.cantaloupe.http.Header; import edu.illinois.library.cantaloupe.http.Headers; -import jakarta.servlet.http.Cookie; -import jakarta.servlet.http.HttpServletResponse; +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletResponse; import java.io.PrintWriter; import java.util.Collection; import java.util.Locale; diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/HTTPStreamFactoryTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/HTTPStreamFactoryTest.java index 7fa0205f3..a1017179a 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/HTTPStreamFactoryTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/HTTPStreamFactoryTest.java @@ -15,8 +15,8 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import javax.imageio.stream.ImageInputStream; import java.io.InputStream; import java.util.Map; diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java index a30046c2a..b22ffe272 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java @@ -17,8 +17,8 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/OkHttpHTTPImageInputStreamClientTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/OkHttpHTTPImageInputStreamClientTest.java index 02a362a08..40617ea81 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/OkHttpHTTPImageInputStreamClientTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/OkHttpHTTPImageInputStreamClientTest.java @@ -10,8 +10,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import static org.junit.jupiter.api.Assertions.*; From e16b27b7ac4e9095abdab6926ec76888d5d44ae4 Mon Sep 17 00:00:00 2001 From: angelahuqing Date: Wed, 8 May 2024 14:58:54 -0700 Subject: [PATCH 082/106] Update Jakarta to Javax and remove UriCompliance --- .../cantaloupe/ApplicationContextListener.java | 4 ++-- .../library/cantaloupe/ApplicationServer.java | 3 --- .../illinois/library/cantaloupe/http/Server.java | 3 --- .../codec/IIOProviderContextListener.java | 4 ++-- .../resource/ByteArrayServletOutputStream.java | 4 ++-- .../resource/MockHttpServletRequest.java | 14 +++++++------- 6 files changed, 13 insertions(+), 19 deletions(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/ApplicationContextListener.java b/src/main/java/edu/illinois/library/cantaloupe/ApplicationContextListener.java index 0071991a2..c7252caa8 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/ApplicationContextListener.java +++ b/src/main/java/edu/illinois/library/cantaloupe/ApplicationContextListener.java @@ -13,8 +13,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import jakarta.servlet.ServletContextEvent; -import jakarta.servlet.ServletContextListener; +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; import javax.script.ScriptEngineManager; import java.util.stream.Collectors; diff --git a/src/main/java/edu/illinois/library/cantaloupe/ApplicationServer.java b/src/main/java/edu/illinois/library/cantaloupe/ApplicationServer.java index a35c07c7f..82b9ef8d0 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/ApplicationServer.java +++ b/src/main/java/edu/illinois/library/cantaloupe/ApplicationServer.java @@ -6,7 +6,6 @@ import edu.illinois.library.cantaloupe.resource.FileServlet; import edu.illinois.library.cantaloupe.resource.HandlerServlet; import org.eclipse.jetty.alpn.server.ALPNServerConnectionFactory; -import org.eclipse.jetty.http.UriCompliance; import org.eclipse.jetty.http2.HTTP2Cipher; import org.eclipse.jetty.http2.server.HTTP2CServerConnectionFactory; import org.eclipse.jetty.http2.server.HTTP2ServerConnectionFactory; @@ -266,7 +265,6 @@ public void start() throws Exception { // HTTP/2. if (isHTTPEnabled()) { HttpConfiguration config = new HttpConfiguration(); - config.setUriCompliance(UriCompliance.LEGACY); HttpConnectionFactory http1 = new HttpConnectionFactory(config); HTTP2CServerConnectionFactory http2 = @@ -282,7 +280,6 @@ public void start() throws Exception { // Initialize the HTTPS server. if (isHTTPSEnabled()) { HttpConfiguration config = new HttpConfiguration(); - config.setUriCompliance(UriCompliance.LEGACY); config.setSecureScheme("https"); config.setSecurePort(getHTTPSPort()); config.addCustomizer(new SecureRequestCustomizer()); diff --git a/src/main/java/edu/illinois/library/cantaloupe/http/Server.java b/src/main/java/edu/illinois/library/cantaloupe/http/Server.java index ba049467e..a22f8196d 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/http/Server.java +++ b/src/main/java/edu/illinois/library/cantaloupe/http/Server.java @@ -2,7 +2,6 @@ import edu.illinois.library.cantaloupe.util.SocketUtils; import org.eclipse.jetty.alpn.server.ALPNServerConnectionFactory; -import org.eclipse.jetty.http.UriCompliance; import org.eclipse.jetty.http2.HTTP2Cipher; import org.eclipse.jetty.http2.server.HTTP2CServerConnectionFactory; import org.eclipse.jetty.http2.server.HTTP2ServerConnectionFactory; @@ -74,7 +73,6 @@ private void initializeServer() { ServerConnector connector; HttpConfiguration config = new HttpConfiguration(); - config.setUriCompliance(UriCompliance.LEGACY); HttpConnectionFactory http1 = new HttpConnectionFactory(config); HTTP2CServerConnectionFactory http2c = @@ -97,7 +95,6 @@ private void initializeServer() { // Initialize HTTPS. if (isHTTPS1Enabled || isHTTPS2Enabled) { config = new HttpConfiguration(); - config.setUriCompliance(UriCompliance.LEGACY); config.setSecureScheme("https"); config.addCustomizer(new SecureRequestCustomizer()); diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/IIOProviderContextListener.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/IIOProviderContextListener.java index 209ed6028..a85b285f9 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/IIOProviderContextListener.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/IIOProviderContextListener.java @@ -4,8 +4,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import jakarta.servlet.ServletContextEvent; -import jakarta.servlet.ServletContextListener; +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; import javax.imageio.ImageIO; import javax.imageio.spi.IIORegistry; import javax.imageio.spi.ServiceRegistry; diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/ByteArrayServletOutputStream.java b/src/test/java/edu/illinois/library/cantaloupe/resource/ByteArrayServletOutputStream.java index 123bc2120..99f8486ec 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/ByteArrayServletOutputStream.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/ByteArrayServletOutputStream.java @@ -1,7 +1,7 @@ package edu.illinois.library.cantaloupe.resource; -import jakarta.servlet.ServletOutputStream; -import jakarta.servlet.WriteListener; +import javax.servlet.ServletOutputStream; +import javax.servlet.WriteListener; import java.io.ByteArrayOutputStream; import java.io.IOException; diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletRequest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletRequest.java index 3b72c90f5..ef6d5926a 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletRequest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletRequest.java @@ -1,12 +1,12 @@ package edu.illinois.library.cantaloupe.resource; -import jakarta.servlet.AsyncContext; -import jakarta.servlet.DispatcherType; -import jakarta.servlet.RequestDispatcher; -import jakarta.servlet.ServletContext; -import jakarta.servlet.ServletInputStream; -import jakarta.servlet.ServletRequest; -import jakarta.servlet.ServletResponse; +import javax.servlet.AsyncContext; +import javax.servlet.DispatcherType; +import javax.servlet.RequestDispatcher; +import javax.servlet.ServletContext; +import javax.servlet.ServletInputStream; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; import javax.servlet.http.Cookie; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; From 4f4bdb961f3380093ef91c93dd46347bf6367d13 Mon Sep 17 00:00:00 2001 From: angelahuqing Date: Wed, 8 May 2024 15:07:05 -0700 Subject: [PATCH 083/106] Replace isCompletelyWritten()with isComplete() which exists in CompletableOutputStream --- .../java/edu/illinois/library/cantaloupe/cache/JdbcCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java index 4d882deb1..82e355bd7 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java @@ -90,7 +90,7 @@ public void close() throws IOException { LOGGER.debug("Closing stream for {}", ops); PreparedStatement statement = null; try { - if (isCompletelyWritten()) { + if (isComplete()) { blobOutputStream.close(); final Configuration config = Configuration.getInstance(); final String sql = String.format( From 553224707fe23335c35a810efe7b54f6e5f88b71 Mon Sep 17 00:00:00 2001 From: angelahuqing Date: Thu, 9 May 2024 13:55:11 -0700 Subject: [PATCH 084/106] Remove isCompletelyWritten() and replace with isComplete() --- .../library/cantaloupe/cache/S3Cache.java | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java index d01117de1..1ef50e7e0 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java @@ -62,6 +62,159 @@ */ class S3Cache implements DerivativeCache { +<<<<<<< HEAD +======= + /** + *

      Wraps a {@link ByteArrayOutputStream} for upload to S3.

      + * + *

      N.B.: S3 does not allow uploads without a {@code Content-Length} + * header, which cannot be provided when streaming an unknown amount of + * data (which this class is going to be doing all the time). From the + * documentation of {@link PutObjectRequest}:

      + * + *
      "When uploading directly from an input stream, content + * length must be specified before data can be uploaded to Amazon S3. If + * not provided, the library will have to buffer the contents of the input + * stream in order to calculate it. Amazon S3 explicitly requires that the + * content length be sent in the request headers before any of the data is + * sent."
      + * + *

      Since it's not possible to write an {@link OutputStream} of unknown + * length to the S3 client as the {@link Cache} interface requires, this + * class buffers written data in a byte array before uploading it to S3 + * upon closure. (The upload is submitted to the + * {@link ThreadPool#getInstance() application thread pool} in order for + * {@link #close()} to be able to return immediately.)

      + */ + private static class S3OutputStream extends CompletableOutputStream { + + private final ByteArrayOutputStream bufferStream = + new ByteArrayOutputStream(); + private final S3Client client; + private final String bucketName; + private final String objectKey; + private final String contentType; + + /** + * @param client S3 client. + * @param bucketName S3 bucket name. + * @param objectKey S3 object key. + * @param contentType Media type. + */ + S3OutputStream(final S3Client client, + final String bucketName, + final String objectKey, + final String contentType) { + this.client = client; + this.bucketName = bucketName; + this.objectKey = objectKey; + this.contentType = contentType; + } + + @Override + public void close() throws IOException { + try { + bufferStream.close(); + byte[] data = bufferStream.toByteArray(); + if (isComplete()) { + // At this point, the client has received all image data, + // but it is still waiting for the connection to close. + // Uploading in a separate thread will allow this to happen + // immediately. + ThreadPool.getInstance().submit(new S3Upload( + client, data, bucketName, objectKey, + contentType, null)); + } + } finally { + super.close(); + } + } + + @Override + public void flush() throws IOException { + bufferStream.flush(); + } + + @Override + public void write(int b) { + bufferStream.write(b); + } + + @Override + public void write(byte[] b) throws IOException { + bufferStream.write(b); + } + + @Override + public void write(byte[] b, int off, int len) { + bufferStream.write(b, off, len); + } + + } + + private static class S3Upload implements Runnable { + + private static final Logger UPLOAD_LOGGER = + LoggerFactory.getLogger(S3Upload.class); + + private final String bucketName, contentEncoding, contentType, objectKey; + private final byte[] data; + private final S3Client client; + + /** + * @param client S3 client. + * @param data Data to upload. + * @param bucketName S3 bucket name. + * @param objectKey S3 object key. + * @param contentType Media type. + * @param contentEncoding Content encoding. May be {@code null}. + */ + S3Upload(S3Client client, + byte[] data, + String bucketName, + String objectKey, + String contentType, + String contentEncoding) { + this.client = client; + this.bucketName = bucketName; + this.data = data; + this.contentType = contentType; + this.contentEncoding = contentEncoding; + this.objectKey = objectKey; + } + + @Override + public void run() { + if (data.length > 0) { + PutObjectRequest request = PutObjectRequest.builder() + .bucket(bucketName) + .key(objectKey) + .contentType(contentType) + .contentEncoding(contentEncoding) + .build(); + final Stopwatch watch = new Stopwatch(); + + UPLOAD_LOGGER.debug("Uploading {} bytes to {} in bucket {}", + data.length, request.key(), request.bucket()); + + try (ByteArrayInputStream is = new ByteArrayInputStream(data)) { + client.putObject(request, + RequestBody.fromInputStream(is, data.length)); + } catch (IOException e) { + UPLOAD_LOGGER.warn(e.getMessage(), e); + } + + UPLOAD_LOGGER.trace("Wrote {} bytes to {} in bucket {} in {}", + data.length, request.key(), request.bucket(), + watch); + } else { + UPLOAD_LOGGER.trace("No data to upload; returning"); + } + } + + } + +>>>>>>> 8a7e3cfe7 (Remove isCompletelyWritten() and replace with isComplete()) private static final Logger LOGGER = LoggerFactory.getLogger(S3Cache.class); From 243860077bea69c8a13f1c1f26507df1b76f4877 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Tue, 16 Jul 2024 19:56:43 +0100 Subject: [PATCH 085/106] Jetty 11 requires jakarta links --- .../source/OkHttpHTTPImageInputStreamClientTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/OkHttpHTTPImageInputStreamClientTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/OkHttpHTTPImageInputStreamClientTest.java index 40617ea81..02a362a08 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/OkHttpHTTPImageInputStreamClientTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/OkHttpHTTPImageInputStreamClientTest.java @@ -10,8 +10,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import static org.junit.jupiter.api.Assertions.*; From df355d7110dec616c7cbf5d472eb35bde6dcbaf5 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Tue, 16 Jul 2024 21:31:07 +0100 Subject: [PATCH 086/106] Jetty 11 requires jakarta.servlet --- .../ApplicationContextListener.java | 4 +-- .../codec/IIOProviderContextListener.java | 4 +-- .../cantaloupe/resource/AbstractResource.java | 4 +-- .../cantaloupe/resource/FileServlet.java | 6 ++--- .../cantaloupe/resource/HandlerServlet.java | 6 ++--- .../library/cantaloupe/resource/Request.java | 2 +- .../library/cantaloupe/http/ServerTest.java | 4 +-- .../ByteArrayServletOutputStream.java | 4 +-- .../resource/MockHttpServletRequest.java | 26 +++++++++---------- .../resource/MockHttpServletResponse.java | 4 +-- .../source/HTTPStreamFactoryTest.java | 4 +-- .../cantaloupe/source/HttpSourceTest.java | 4 +-- 12 files changed, 36 insertions(+), 36 deletions(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/ApplicationContextListener.java b/src/main/java/edu/illinois/library/cantaloupe/ApplicationContextListener.java index c7252caa8..0071991a2 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/ApplicationContextListener.java +++ b/src/main/java/edu/illinois/library/cantaloupe/ApplicationContextListener.java @@ -13,8 +13,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.ServletContextEvent; -import javax.servlet.ServletContextListener; +import jakarta.servlet.ServletContextEvent; +import jakarta.servlet.ServletContextListener; import javax.script.ScriptEngineManager; import java.util.stream.Collectors; diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/IIOProviderContextListener.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/IIOProviderContextListener.java index a85b285f9..209ed6028 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/IIOProviderContextListener.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/IIOProviderContextListener.java @@ -4,8 +4,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.ServletContextEvent; -import javax.servlet.ServletContextListener; +import jakarta.servlet.ServletContextEvent; +import jakarta.servlet.ServletContextListener; import javax.imageio.ImageIO; import javax.imageio.spi.IIORegistry; import javax.imageio.spi.ServiceRegistry; diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/AbstractResource.java b/src/main/java/edu/illinois/library/cantaloupe/resource/AbstractResource.java index 20181b298..7b2c04e8d 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/AbstractResource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/AbstractResource.java @@ -21,7 +21,7 @@ import edu.illinois.library.cantaloupe.util.StringUtils; import org.slf4j.Logger; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; @@ -41,7 +41,7 @@ * more of the HTTP-method-specific methods {@link #doGET()} etc., and may * optionally use {@link #doInit()} and {@link #destroy()}.

      * - *

      Unlike {@link javax.servlet.http.HttpServlet}s, instances are only used + *

      Unlike {@link jakarta.servlet.http.HttpServlet}s, instances are only used * once and not shared across threads.

      */ public abstract class AbstractResource { diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/FileServlet.java b/src/main/java/edu/illinois/library/cantaloupe/resource/FileServlet.java index cbad65f02..94551c415 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/FileServlet.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/FileServlet.java @@ -1,8 +1,8 @@ package edu.illinois.library.cantaloupe.resource; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/HandlerServlet.java b/src/main/java/edu/illinois/library/cantaloupe/resource/HandlerServlet.java index 2223870ec..b84025a3d 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/HandlerServlet.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/HandlerServlet.java @@ -8,9 +8,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.PrintWriter; import java.util.List; diff --git a/src/main/java/edu/illinois/library/cantaloupe/resource/Request.java b/src/main/java/edu/illinois/library/cantaloupe/resource/Request.java index 55588eab4..9e36c9a38 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/resource/Request.java +++ b/src/main/java/edu/illinois/library/cantaloupe/resource/Request.java @@ -6,7 +6,7 @@ import edu.illinois.library.cantaloupe.http.Query; import edu.illinois.library.cantaloupe.http.Reference; -import javax.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletRequest; import java.io.IOException; import java.io.InputStream; import java.util.Enumeration; diff --git a/src/test/java/edu/illinois/library/cantaloupe/http/ServerTest.java b/src/test/java/edu/illinois/library/cantaloupe/http/ServerTest.java index 84c79a215..1e6002d58 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/http/ServerTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/http/ServerTest.java @@ -9,8 +9,8 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import static org.junit.jupiter.api.Assertions.*; diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/ByteArrayServletOutputStream.java b/src/test/java/edu/illinois/library/cantaloupe/resource/ByteArrayServletOutputStream.java index 99f8486ec..123bc2120 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/ByteArrayServletOutputStream.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/ByteArrayServletOutputStream.java @@ -1,7 +1,7 @@ package edu.illinois.library.cantaloupe.resource; -import javax.servlet.ServletOutputStream; -import javax.servlet.WriteListener; +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.WriteListener; import java.io.ByteArrayOutputStream; import java.io.IOException; diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletRequest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletRequest.java index ef6d5926a..1bdb796ab 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletRequest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletRequest.java @@ -1,18 +1,18 @@ package edu.illinois.library.cantaloupe.resource; -import javax.servlet.AsyncContext; -import javax.servlet.DispatcherType; -import javax.servlet.RequestDispatcher; -import javax.servlet.ServletContext; -import javax.servlet.ServletInputStream; -import javax.servlet.ServletRequest; -import javax.servlet.ServletResponse; -import javax.servlet.http.Cookie; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.servlet.http.HttpSession; -import javax.servlet.http.HttpUpgradeHandler; -import javax.servlet.http.Part; +import jakarta.servlet.AsyncContext; +import jakarta.servlet.DispatcherType; +import jakarta.servlet.RequestDispatcher; +import jakarta.servlet.ServletContext; +import jakarta.servlet.ServletInputStream; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import jakarta.servlet.http.Cookie; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpSession; +import jakarta.servlet.http.HttpUpgradeHandler; +import jakarta.servlet.http.Part; import java.io.BufferedReader; import java.security.Principal; import java.util.Collection; diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletResponse.java b/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletResponse.java index 749e314f3..638cd4599 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletResponse.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/MockHttpServletResponse.java @@ -3,8 +3,8 @@ import edu.illinois.library.cantaloupe.http.Header; import edu.illinois.library.cantaloupe.http.Headers; -import javax.servlet.http.Cookie; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.Cookie; +import jakarta.servlet.http.HttpServletResponse; import java.io.PrintWriter; import java.util.Collection; import java.util.Locale; diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/HTTPStreamFactoryTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/HTTPStreamFactoryTest.java index a1017179a..7fa0205f3 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/HTTPStreamFactoryTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/HTTPStreamFactoryTest.java @@ -15,8 +15,8 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import javax.imageio.stream.ImageInputStream; import java.io.InputStream; import java.util.Map; diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java index b22ffe272..a30046c2a 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java @@ -17,8 +17,8 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; From b02b98dd051a6e0d05ea077503c81d7244ee2b3d Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 17 Jul 2024 00:31:57 +0100 Subject: [PATCH 087/106] Fixing merge issue --- .../library/cantaloupe/cache/S3Cache.java | 153 ------------------ 1 file changed, 153 deletions(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java index 1ef50e7e0..d01117de1 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/S3Cache.java @@ -62,159 +62,6 @@ */ class S3Cache implements DerivativeCache { -<<<<<<< HEAD -======= - /** - *

      Wraps a {@link ByteArrayOutputStream} for upload to S3.

      - * - *

      N.B.: S3 does not allow uploads without a {@code Content-Length} - * header, which cannot be provided when streaming an unknown amount of - * data (which this class is going to be doing all the time). From the - * documentation of {@link PutObjectRequest}:

      - * - *
      "When uploading directly from an input stream, content - * length must be specified before data can be uploaded to Amazon S3. If - * not provided, the library will have to buffer the contents of the input - * stream in order to calculate it. Amazon S3 explicitly requires that the - * content length be sent in the request headers before any of the data is - * sent."
      - * - *

      Since it's not possible to write an {@link OutputStream} of unknown - * length to the S3 client as the {@link Cache} interface requires, this - * class buffers written data in a byte array before uploading it to S3 - * upon closure. (The upload is submitted to the - * {@link ThreadPool#getInstance() application thread pool} in order for - * {@link #close()} to be able to return immediately.)

      - */ - private static class S3OutputStream extends CompletableOutputStream { - - private final ByteArrayOutputStream bufferStream = - new ByteArrayOutputStream(); - private final S3Client client; - private final String bucketName; - private final String objectKey; - private final String contentType; - - /** - * @param client S3 client. - * @param bucketName S3 bucket name. - * @param objectKey S3 object key. - * @param contentType Media type. - */ - S3OutputStream(final S3Client client, - final String bucketName, - final String objectKey, - final String contentType) { - this.client = client; - this.bucketName = bucketName; - this.objectKey = objectKey; - this.contentType = contentType; - } - - @Override - public void close() throws IOException { - try { - bufferStream.close(); - byte[] data = bufferStream.toByteArray(); - if (isComplete()) { - // At this point, the client has received all image data, - // but it is still waiting for the connection to close. - // Uploading in a separate thread will allow this to happen - // immediately. - ThreadPool.getInstance().submit(new S3Upload( - client, data, bucketName, objectKey, - contentType, null)); - } - } finally { - super.close(); - } - } - - @Override - public void flush() throws IOException { - bufferStream.flush(); - } - - @Override - public void write(int b) { - bufferStream.write(b); - } - - @Override - public void write(byte[] b) throws IOException { - bufferStream.write(b); - } - - @Override - public void write(byte[] b, int off, int len) { - bufferStream.write(b, off, len); - } - - } - - private static class S3Upload implements Runnable { - - private static final Logger UPLOAD_LOGGER = - LoggerFactory.getLogger(S3Upload.class); - - private final String bucketName, contentEncoding, contentType, objectKey; - private final byte[] data; - private final S3Client client; - - /** - * @param client S3 client. - * @param data Data to upload. - * @param bucketName S3 bucket name. - * @param objectKey S3 object key. - * @param contentType Media type. - * @param contentEncoding Content encoding. May be {@code null}. - */ - S3Upload(S3Client client, - byte[] data, - String bucketName, - String objectKey, - String contentType, - String contentEncoding) { - this.client = client; - this.bucketName = bucketName; - this.data = data; - this.contentType = contentType; - this.contentEncoding = contentEncoding; - this.objectKey = objectKey; - } - - @Override - public void run() { - if (data.length > 0) { - PutObjectRequest request = PutObjectRequest.builder() - .bucket(bucketName) - .key(objectKey) - .contentType(contentType) - .contentEncoding(contentEncoding) - .build(); - final Stopwatch watch = new Stopwatch(); - - UPLOAD_LOGGER.debug("Uploading {} bytes to {} in bucket {}", - data.length, request.key(), request.bucket()); - - try (ByteArrayInputStream is = new ByteArrayInputStream(data)) { - client.putObject(request, - RequestBody.fromInputStream(is, data.length)); - } catch (IOException e) { - UPLOAD_LOGGER.warn(e.getMessage(), e); - } - - UPLOAD_LOGGER.trace("Wrote {} bytes to {} in bucket {} in {}", - data.length, request.key(), request.bucket(), - watch); - } else { - UPLOAD_LOGGER.trace("No data to upload; returning"); - } - } - - } - ->>>>>>> 8a7e3cfe7 (Remove isCompletelyWritten() and replace with isComplete()) private static final Logger LOGGER = LoggerFactory.getLogger(S3Cache.class); From 6d8c554ea65063d2e6858ca119510fa0094dd750 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 17 Jul 2024 00:37:08 +0100 Subject: [PATCH 088/106] Fixing compile issue --- .../java/edu/illinois/library/cantaloupe/cache/JdbcCache.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java b/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java index 82e355bd7..1fa4ffa64 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/JdbcCache.java @@ -80,7 +80,6 @@ private class ImageBlobOutputStream extends CompletableOutputStream { DERIVATIVE_IMAGE_TABLE_LAST_ACCESSED_COLUMN); LOGGER.trace(sql); - final Blob blob = connection.createBlob(); blob = connection.createBlob(); blobOutputStream = blob.setBinaryStream(1); } From 0169798c936df4b4fe10ded2c4cfb25c4feba246 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 17 Jul 2024 23:31:15 +0100 Subject: [PATCH 089/106] Fixing testPurgeInvalidWithKeyPrefix with async cache --- .../library/cantaloupe/cache/S3CacheTest.java | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java index 6f00ec903..323bdb9e6 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java @@ -25,7 +25,6 @@ import software.amazon.awssdk.services.s3.model.PutObjectRequest; import java.io.ByteArrayInputStream; -import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -131,6 +130,25 @@ private static Service getService() { return Service.forKey(testConfig.getString(ConfigurationConstants.S3_SERVICE.getKey())); } + private void uploadDerivative(OperationList ops1, Path fixture) throws Exception { + CompletableOutputStream outputStream = null; + try { + outputStream = instance.newDerivativeImageOutputStream(ops1); + if (outputStream instanceof S3MultipartAsyncOutputStream) { + ((S3MultipartAsyncOutputStream)outputStream).observer = this; + } + Files.copy(fixture, outputStream); + outputStream.setComplete(true); + } finally { + outputStream.close(); + } + if (outputStream instanceof S3MultipartAsyncOutputStream) { + synchronized (outputStream) { + outputStream.wait(); + } + } + } + @BeforeEach public void setUp() throws Exception { super.setUp(); @@ -298,12 +316,8 @@ void testPurgeWithKeyPrefix() throws Exception { } // Add a cached derivative image - try (CompletableOutputStream outputStream = - instance.newDerivativeImageOutputStream(opList)) { - Path fixture = TestUtil.getImage(IMAGE); - Files.copy(fixture, outputStream); - outputStream.setComplete(true); - } + Path fixture = TestUtil.getImage(IMAGE); + uploadDerivative(opList, fixture); // Add a cached info instance.put(identifier, info); @@ -344,6 +358,8 @@ void testPurgeInvalid() throws Exception { super.testPurgeInvalid(); } + + @Test void testPurgeInvalidWithKeyPrefix() throws Exception { final String prefix = "prefix/"; @@ -376,11 +392,7 @@ void testPurgeInvalidWithKeyPrefix() throws Exception { .withOperations(new Encode(Format.get("jpg"))) .build(); Path fixture = TestUtil.getImage(id1.toString()); - try (CompletableOutputStream outputStream = - instance.newDerivativeImageOutputStream(ops1)) { - Files.copy(fixture, outputStream); - outputStream.setComplete(true); - } + uploadDerivative(ops1, fixture); // add a cached Info Info info1 = new Info(); From 6e38c764f4df935c8c93553fa82388d7a1e6ea97 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Thu, 18 Jul 2024 00:55:43 +0100 Subject: [PATCH 090/106] Fixing purgeInvalid to use async upload --- .../library/cantaloupe/cache/S3CacheTest.java | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java index 323bdb9e6..aba717b4d 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java @@ -355,7 +355,54 @@ void testPurgeWithKeyPrefix() throws Exception { @Override void testPurgeInvalid() throws Exception { assumeFalse(SystemUtils.IS_OS_WINDOWS); // TODO: this fails in Windows sometimes - super.testPurgeInvalid(); + + DerivativeCache instance = newInstance(); + Identifier id1 = new Identifier(IMAGE); + OperationList ops1 = OperationList.builder() + .withIdentifier(id1) + .withOperations(new Encode(Format.get("jpg"))) + .build(); + Info info1 = new Info(); + Configuration.getInstance().setProperty(Key.DERIVATIVE_CACHE_TTL, 2); + + // add an image + Path fixture = TestUtil.getImage(id1.toString()); + uploadDerivative(ops1, fixture); + + // add an Info + instance.put(id1, info1); + + // assert that they've been added + assertNotNull(instance.getInfo(id1)); + assertExists(instance, ops1); + + // wait for them to invalidate + Thread.sleep(2100); + + // add another image + Path fixture2 = TestUtil.getImage("gif-rgb-64x56x8.gif"); + OperationList ops2 = OperationList.builder() + .withIdentifier(new Identifier(fixture2.getFileName().toString())) + .withOperations(new Encode(Format.get("jpg"))) + .build(); + + uploadDerivative(ops2, fixture2); + + // add another info + Identifier id2 = new Identifier("cats"); + instance.put(id2, new Info()); + + // assert that they've been added + assertNotNull(instance.getInfo(id2)); + assertExists(instance, ops2); + + instance.purgeInvalid(); + + // assert that one image and one info have been purged + assertFalse(instance.getInfo(id1).isPresent()); + assertTrue(instance.getInfo(id2).isPresent()); + assertNotExists(instance, ops1); + assertExists(instance, ops2); } From 7f8d3cbdd28e90ae34acf87c62a5e8c8068ae8bc Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Thu, 18 Jul 2024 10:29:50 +0100 Subject: [PATCH 091/106] Fixing testPurge --- .../library/cantaloupe/cache/S3CacheTest.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java b/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java index aba717b4d..d1184a719 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/cache/S3CacheTest.java @@ -25,6 +25,7 @@ import software.amazon.awssdk.services.s3.model.PutObjectRequest; import java.io.ByteArrayInputStream; +import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -286,6 +287,51 @@ void testNewDerivativeImageInputStreamUpdatesLastModifiedTime() } /* purge() */ + @Test + @Override + void testPurge() throws Exception { + DerivativeCache instance = newInstance(); + Identifier identifier = new Identifier(IMAGE); + OperationList opList = OperationList.builder() + .withIdentifier(identifier) + .withOperations(new Encode(Format.get("jpg"))) + .build(); + Info info = new Info(); + + // assert that a particular image doesn't exist + try (InputStream is = instance.newDerivativeImageInputStream(opList)) { + assertNull(is); + } + + // assert that a particular info doesn't exist + assertFalse(instance.getInfo(identifier).isPresent()); + + // add the image + + Path fixture = TestUtil.getImage(IMAGE); + uploadDerivative(opList, fixture); + + // add the info + instance.put(identifier, info); + + Thread.sleep(ASYNC_WAIT); + + // assert that they've been added + assertExists(instance, opList); + assertNotNull(instance.getInfo(identifier)); + + // purge everything + instance.purge(); + + // Allow time for purge but not as long as upload + Thread.sleep(ASYNC_WAIT / 2); + + // assert that the info has been purged + assertFalse(instance.getInfo(identifier).isPresent()); + + // assert that the image has been purged + assertNotExists(instance, opList); + } @Test void testPurgeWithKeyPrefix() throws Exception { From fc4d1e9c4f9af4e641ac5cc25ba2d0dd1c610f81 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Thu, 18 Jul 2024 10:56:17 +0100 Subject: [PATCH 092/106] Ensuring part numbers are sequential --- .../cantaloupe/cache/S3MultipartAsyncOutputStream.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStream.java b/src/main/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStream.java index 56990bf74..45a452fb0 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStream.java +++ b/src/main/java/edu/illinois/library/cantaloupe/cache/S3MultipartAsyncOutputStream.java @@ -117,8 +117,7 @@ private class PartUploader implements Runnable { @Override public void run() { try { - final int partNumber = partIndex + 1; - + final int partNumber = partIndex++; UploadPartRequest uploadPartRequest = UploadPartRequest.builder() .bucket(bucket) .key(key) @@ -229,7 +228,9 @@ public void run() { private boolean requestCreated; private String uploadID; - private int partIndex; + // Part number must start with 1 + // minio will hang if this is 0 + private int partIndex = 1; private long indexWithinPart; /** For an instance to wait for an upload notification during testing. */ @@ -315,7 +316,6 @@ private void uploadPartIfNecessary() { IOUtils.closeQuietly(currentPart); currentPart = null; indexWithinPart = 0; - partIndex++; } } From 30cbf8632c4c979a2be844920ab475482bab52a5 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Thu, 18 Jul 2024 11:09:04 +0100 Subject: [PATCH 093/106] Fixing empty namespace issue with XMP --- .../cantaloupe/image/xmp/MapReader.java | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/xmp/MapReader.java b/src/main/java/edu/illinois/library/cantaloupe/image/xmp/MapReader.java index 51b608fe6..a2e888d8d 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/image/xmp/MapReader.java +++ b/src/main/java/edu/illinois/library/cantaloupe/image/xmp/MapReader.java @@ -59,13 +59,22 @@ public final class MapReader { public MapReader(String xmp) throws IOException { RIOT.init(); this.model = ModelFactory.createDefaultModel(); - try (StringReader reader = new StringReader(xmp)) { - model.read(reader, null, "RDF/XML"); - } catch (RiotException | NullPointerException e) { - // The XMP string may be invalid RDF/XML, or there may be a bug - // in Jena (that would be the NPE). Not much we can do. - throw new IOException(e); - } + if (xmp != null) { + String base = null; + if (xmp.indexOf("rdf:about=''") != -1 || xmp.indexOf("rdf:about=\"\"") != -1) { + // Version 4.8+ of jena requires a rdf:about link to not be empty + base = "http://example.com"; + } + try (StringReader reader = new StringReader(xmp)) { + model.read(reader, base, "RDF/XML"); + } catch (RiotException | NullPointerException e) { + // The XMP string may be invalid RDF/XML, or there may be a bug + // in Jena (that would be the NPE). Not much we can do. + throw new IOException(e); + } + } else { + throw new IOException(new NullPointerException("XMP not supplied")); + } } /** From 58018db230d93f2bbbc14279d90d8e55957ebe96 Mon Sep 17 00:00:00 2001 From: Camille Hodoul Date: Tue, 30 Jul 2024 16:35:00 +0200 Subject: [PATCH 094/106] Upgrade maven-assembly-plugin to 3.7.1 - CVE-2023-37460 This bumps plexus-archiver to 4.9.2, which fixes CVE-2023-37460 (starting from 4.8) see: - https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12317220&version=12353243 - https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12317220&version=12354406 - https://nvd.nist.gov/vuln/detail/CVE-2023-37460 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 6141292e1..63d937c26 100644 --- a/pom.xml +++ b/pom.xml @@ -371,7 +371,7 @@ org.apache.maven.plugins maven-assembly-plugin - 3.6.0 + 3.7.1 maven-plugin @@ -503,7 +503,7 @@ maven-assembly-plugin - 3.6.0 + 3.7.1 package From 09b6b54708d09bc7b5b7fce9004d99c805ad5297 Mon Sep 17 00:00:00 2001 From: Diego Pino Navarro Date: Thu, 1 Aug 2024 17:30:02 -0400 Subject: [PATCH 095/106] Update ci.yml for docker compose on ubuntu-latest See #682 --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9e6dfe5a..0b073fde3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,13 +13,13 @@ jobs: uses: actions/checkout@v2 - name: Test in Linux JDK 11 if: matrix.os == 'ubuntu-latest' && matrix.java == 'jdk11' - run: docker-compose -f docker/Linux-JDK11/docker-compose.yml up --build --exit-code-from cantaloupe + run: docker compose -f docker/Linux-JDK11/docker-compose.yml up --build --exit-code-from cantaloupe - name: Test in Linux JDK 18 if: matrix.os == 'ubuntu-latest' && matrix.java == 'jdk18' - run: docker-compose -f docker/Linux-JDK18/docker-compose.yml up --build --exit-code-from cantaloupe + run: docker compose -f docker/Linux-JDK18/docker-compose.yml up --build --exit-code-from cantaloupe - name: Test in Linux GraalVM if: matrix.os == 'ubuntu-latest' && matrix.java == 'graalvm' - run: docker-compose -f docker/Linux-GraalVM20/docker-compose.yml up --build --exit-code-from cantaloupe + run: docker compose -f docker/Linux-GraalVM20/docker-compose.yml up --build --exit-code-from cantaloupe - name: Test in Windows JDK 11 if: matrix.os == 'windows-latest' && matrix.java == 'jdk11' run: docker-compose -f docker/Windows-JDK11/docker-compose.yml up --build --exit-code-from cantaloupe From f5315fdf5851138780d0999156cd84cc9d1ce12d Mon Sep 17 00:00:00 2001 From: Adam Vessey Date: Tue, 13 Aug 2024 12:11:41 -0300 Subject: [PATCH 096/106] Attempt moving over to the newer `docker compose`. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b073fde3..02884e2a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,9 +22,9 @@ jobs: run: docker compose -f docker/Linux-GraalVM20/docker-compose.yml up --build --exit-code-from cantaloupe - name: Test in Windows JDK 11 if: matrix.os == 'windows-latest' && matrix.java == 'jdk11' - run: docker-compose -f docker/Windows-JDK11/docker-compose.yml up --build --exit-code-from cantaloupe + run: docker compose -f docker/Windows-JDK11/docker-compose.yml up --build --exit-code-from cantaloupe - name: Test in Windows JDK 18 if: matrix.os == 'windows-latest' && matrix.java == 'jdk18' - run: docker-compose -f docker/Windows-JDK18/docker-compose.yml up --build --exit-code-from cantaloupe + run: docker compose -f docker/Windows-JDK18/docker-compose.yml up --build --exit-code-from cantaloupe # TODO: Windows+GraalVM From 7547c426dabb342a8adae31f9793c14fb1d073a2 Mon Sep 17 00:00:00 2001 From: Adam Vessey Date: Thu, 6 Jun 2024 11:01:06 -0300 Subject: [PATCH 097/106] Fix some unsafe reads. If they failed to read the number of bytes, the return of `-1` would cause infinite loops, with `-1` always less than the minimum `offset` of `0`. --- .../edu/illinois/library/cantaloupe/image/exif/Reader.java | 5 +---- .../cantaloupe/processor/codec/gif/GIFMetadataReader.java | 6 +----- .../cantaloupe/processor/codec/jpeg/JPEGMetadataReader.java | 6 +----- .../processor/codec/jpeg2000/JPEG2000MetadataReader.java | 6 +----- 4 files changed, 4 insertions(+), 19 deletions(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/image/exif/Reader.java b/src/main/java/edu/illinois/library/cantaloupe/image/exif/Reader.java index bc4261823..1c4ba8037 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/image/exif/Reader.java +++ b/src/main/java/edu/illinois/library/cantaloupe/image/exif/Reader.java @@ -155,10 +155,7 @@ private Directory read(TagSet tagSet) throws IOException { private byte[] readBytes(int length) throws IOException { byte[] data = new byte[length]; - int n, offset = 0; - while ((n = inputStream.read(data, offset, data.length - offset)) < offset) { - offset += n; - } + inputStream.readFully(data); return data; } diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/gif/GIFMetadataReader.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/gif/GIFMetadataReader.java index 1978807d2..14887dad3 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/gif/GIFMetadataReader.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/gif/GIFMetadataReader.java @@ -340,11 +340,7 @@ private void readPlainTextExtension() throws IOException { private byte[] read(int length) throws IOException { byte[] data = new byte[length]; - int n, offset = 0; - while ((n = inputStream.read( - data, offset, data.length - offset)) < offset) { - offset += n; - } + inputStream.readFully(data); return data; } diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/JPEGMetadataReader.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/JPEGMetadataReader.java index b54056c0c..44be18b2d 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/JPEGMetadataReader.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/JPEGMetadataReader.java @@ -296,11 +296,7 @@ private void readAPP14Segment() throws IOException { private byte[] read(int length) throws IOException { byte[] data = new byte[length]; - int n, offset = 0; - while ((n = inputStream.read( - data, offset, data.length - offset)) < offset) { - offset += n; - } + inputStream.readFully(data); return data; } diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg2000/JPEG2000MetadataReader.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg2000/JPEG2000MetadataReader.java index ab9252d2a..24da4bef5 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg2000/JPEG2000MetadataReader.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg2000/JPEG2000MetadataReader.java @@ -434,11 +434,7 @@ private void readCOCSegment() throws IOException { private byte[] read(int length) throws IOException { byte[] data = new byte[length]; - int n, offset = 0; - while ((n = inputStream.read( - data, offset, data.length - offset)) < offset) { - offset += n; - } + inputStream.readFully(data); return data; } From 73119cd675173997ad73f80737da65a633d85600 Mon Sep 17 00:00:00 2001 From: Adam Vessey Date: Wed, 31 Jul 2024 13:18:26 -0300 Subject: [PATCH 098/106] Attempt to wrap JPEG2000 reads to consistently throw SourceFormatException. --- .../jpeg2000/JPEG2000MetadataReader.java | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg2000/JPEG2000MetadataReader.java b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg2000/JPEG2000MetadataReader.java index 24da4bef5..ecf2b947f 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg2000/JPEG2000MetadataReader.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/codec/jpeg2000/JPEG2000MetadataReader.java @@ -8,6 +8,7 @@ import javax.imageio.stream.ImageInputStream; import javax.xml.bind.DatatypeConverter; +import java.io.EOFException; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; @@ -268,23 +269,29 @@ private void readData() throws IOException { throw new IllegalStateException("Source not set"); } - inputStream.mark(); - byte[] bytes = read(JP2_SIGNATURE.length); - if (!Arrays.equals(JP2_SIGNATURE, bytes)) { - String hexStr = DatatypeConverter.printHexBinary(bytes); - throw new SourceFormatException("Invalid signature: " + hexStr + - " (is this a JP2?)"); - } - inputStream.reset(); + try { + inputStream.mark(); + byte[] bytes = read(JP2_SIGNATURE.length); + if (!Arrays.equals(JP2_SIGNATURE, bytes)) { + String hexStr = DatatypeConverter.printHexBinary(bytes); + throw new SourceFormatException("Invalid signature: " + hexStr + + " (is this a JP2?)"); + } + inputStream.reset(); - final Stopwatch watch = new Stopwatch(); - while (readBox() != -1) { - // Read boxes. - isReadAttempted = true; - } + final Stopwatch watch = new Stopwatch(); + + while (readBox() != -1) { + // Read boxes. + isReadAttempted = true; + } - LOGGER.debug("Read in {}: {}", watch, this); + LOGGER.debug("Read in {}: {}", watch, this); + } + catch (EOFException e) { + throw (SourceFormatException)(new SourceFormatException("JP2 appears to be corrupt; encountered EOF.").initCause(e)); + } } private int readBox() throws IOException { From 2bf4e4d379a718487d84f587d0c6e3d669235af6 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Wed, 14 Aug 2024 15:56:17 +0100 Subject: [PATCH 099/106] Adding failing test to check if the fix for PDF is working --- .../codec/jpeg/TurboJPEGImageWriterTest.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/test/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriterTest.java b/src/test/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriterTest.java index 142fb37fc..c17d71c0d 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriterTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/processor/codec/jpeg/TurboJPEGImageWriterTest.java @@ -164,6 +164,28 @@ public void testWriteWithBufferedImage() throws Exception { } } + /** + * Note the TurboJPEGImageWriter.write method is used in the PDFbox processor + */ + @Test + public void testWriteWithBufferedImageYZero() throws Exception { + Path path = TestUtil.getImage("jpg"); + BufferedImage image = ImageIO.read(path.toFile()); + + image = image.getSubimage(0, 10, 10,10); + + try (ByteArrayOutputStream os = new ByteArrayOutputStream()) { + instance.write(image, os); + ImageInputStream is = ImageIO.createImageInputStream(new ByteArrayInputStream(os.toByteArray())); + image = ImageIO.read(is); // also closes the stream + Color color = new Color(image.getRGB(0,0)); + assertEquals(116, color.getRed(), "Expected a particular color pixel but it wasn't found. Potentially the TurboJPEGImageWriter is returning the wrong image. Red value"); + assertEquals(151, color.getGreen(), "Expected a particular color pixel but it wasn't found. Potentially the TurboJPEGImageWriter is returning the wrong image. Green value"); + assertEquals(97, color.getBlue(), "Expected a particular color pixel but it wasn't found. Potentially the TurboJPEGImageWriter is returning the wrong image. Blue value"); + // ImageIO.write(image, "jpg", new java.io.File("/tmp/test.jpg")); + } + } + @Test public void testWriteWithGrayBufferedImage() throws Exception { BufferedImage image = new BufferedImage(50, 50, From 397de41d032e38d43072e1e7c3fe9634d94a0b29 Mon Sep 17 00:00:00 2001 From: Gary Tierney Date: Fri, 28 Oct 2022 11:27:04 +0100 Subject: [PATCH 100/106] Avoid communicating with S3 unless necessary Previously the S3 source would eagerly connect to S3 retrieve information about the object being requested when creating a stream. This change defers retrieval of properties from S3 until needed by the S3StreamFactory, and allows cached files to be served directly from disk with no intermediate S3 lookups. --- .../cantaloupe/source/S3ObjectInfoSupplier.java | 11 +++++++++++ .../illinois/library/cantaloupe/source/S3Source.java | 8 +++++--- .../library/cantaloupe/source/S3StreamFactory.java | 6 ++++-- .../cantaloupe/source/S3StreamFactoryTest.java | 2 +- 4 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 src/main/java/edu/illinois/library/cantaloupe/source/S3ObjectInfoSupplier.java diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/S3ObjectInfoSupplier.java b/src/main/java/edu/illinois/library/cantaloupe/source/S3ObjectInfoSupplier.java new file mode 100644 index 000000000..f48a9651b --- /dev/null +++ b/src/main/java/edu/illinois/library/cantaloupe/source/S3ObjectInfoSupplier.java @@ -0,0 +1,11 @@ +package edu.illinois.library.cantaloupe.source; + +import java.io.IOException; + +/** + * Deferred accessor to {@link S3ObjectInfo}. + */ +@FunctionalInterface +public interface S3ObjectInfoSupplier { + S3ObjectInfo get() throws IOException; +} diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/S3Source.java b/src/main/java/edu/illinois/library/cantaloupe/source/S3Source.java index 93db8621f..67ac60fe3 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/S3Source.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/S3Source.java @@ -422,9 +422,11 @@ private S3ObjectInfo getObjectInfoUsingDelegateStrategy() @Override public StreamFactory newStreamFactory() throws IOException { - S3ObjectInfo info = getObjectInfo(); - info.setLength(getObjectAttributes().length); - return new S3StreamFactory(info); + return new S3StreamFactory(() -> { + S3ObjectInfo info = getObjectInfo(); + info.setLength(getObjectAttributes().length); + return info; + }); } @Override diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/S3StreamFactory.java b/src/main/java/edu/illinois/library/cantaloupe/source/S3StreamFactory.java index d08a0af76..d27d92db3 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/S3StreamFactory.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/S3StreamFactory.java @@ -24,14 +24,15 @@ class S3StreamFactory implements StreamFactory { private static final int DEFAULT_CHUNK_SIZE = 1024 * 512; private static final int DEFAULT_CHUNK_CACHE_SIZE = 1024 * 1024 * 10; - private S3ObjectInfo objectInfo; + private S3ObjectInfoSupplier objectInfo; - S3StreamFactory(S3ObjectInfo objectInfo) { + S3StreamFactory(S3ObjectInfoSupplier objectInfo) { this.objectInfo = objectInfo; } @Override public InputStream newInputStream() throws IOException { + final S3ObjectInfo objectInfo = this.objectInfo.get(); final InputStream responseStream = S3Source.newObjectInputStream(objectInfo); @@ -100,6 +101,7 @@ public ImageInputStream newSeekableStream() throws IOException { LOGGER.debug("newSeekableStream(): using {}-byte chunks", chunkSize); + final S3ObjectInfo objectInfo = this.objectInfo.get(); final S3HTTPImageInputStreamClient client = new S3HTTPImageInputStreamClient(objectInfo); diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/S3StreamFactoryTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/S3StreamFactoryTest.java index c1432a948..3a8082cc3 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/S3StreamFactoryTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/S3StreamFactoryTest.java @@ -109,7 +109,7 @@ public void setUp() throws Exception { info.setKey(FIXTURE_KEY); info.setLength(1584); - instance = new S3StreamFactory(info); + instance = new S3StreamFactory(() -> info); } @Test From 174259587c2cd2eb743ec0015cebddb61d0eac27 Mon Sep 17 00:00:00 2001 From: Justin Coyne Date: Thu, 15 Aug 2024 00:06:58 -0500 Subject: [PATCH 101/106] Upgrade to actions/checkout@v4 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02884e2a4..36db57e9f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ jobs: fail-fast: false steps: - name: Check out the repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Test in Linux JDK 11 if: matrix.os == 'ubuntu-latest' && matrix.java == 'jdk11' run: docker compose -f docker/Linux-JDK11/docker-compose.yml up --build --exit-code-from cantaloupe From d0f4e0e0fb226b896536c4fe5424104abdafac9f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 05:51:09 +0000 Subject: [PATCH 102/106] Bump org.bouncycastle:bcprov-jdk18on from 1.76 to 1.78 Bumps [org.bouncycastle:bcprov-jdk18on](https://github.com/bcgit/bc-java) from 1.76 to 1.78. - [Changelog](https://github.com/bcgit/bc-java/blob/main/docs/releasenotes.html) - [Commits](https://github.com/bcgit/bc-java/commits) --- updated-dependencies: - dependency-name: org.bouncycastle:bcprov-jdk18on dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 63d937c26..489e5b621 100644 --- a/pom.xml +++ b/pom.xml @@ -222,7 +222,7 @@ org.bouncycastle bcprov-jdk18on - 1.76 + 1.78 From aebacc5a1e205f4c90e0b40742d6369d7ce1b8ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 06:11:03 +0000 Subject: [PATCH 103/106] Bump org.eclipse.jetty.http2:http2-server from 11.0.5 to 11.0.17 Bumps org.eclipse.jetty.http2:http2-server from 11.0.5 to 11.0.17. --- updated-dependencies: - dependency-name: org.eclipse.jetty.http2:http2-server dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 489e5b621..bc7f4803a 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ 11 UTF-8 UTF-8 - 11.0.5 + 11.0.17 2.21.4 2.15.2 9.4.3.0 From 95f615bd230d591fb83a9e6e867ffd58178756ac Mon Sep 17 00:00:00 2001 From: Justin Coyne Date: Thu, 15 Aug 2024 01:08:22 -0500 Subject: [PATCH 104/106] Adjust settings for ci builds --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02884e2a4..cb8853f9f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,10 @@ name: CI -on: [push, pull_request] +on: + push: + branches: + - develop + - release/* + pull_request: jobs: build: runs-on: ${{ matrix.os }} @@ -27,4 +32,3 @@ jobs: if: matrix.os == 'windows-latest' && matrix.java == 'jdk18' run: docker compose -f docker/Windows-JDK18/docker-compose.yml up --build --exit-code-from cantaloupe # TODO: Windows+GraalVM - From fd8e2819b4630ed22731e3a0907df4dbe973d905 Mon Sep 17 00:00:00 2001 From: Justin Coyne Date: Thu, 15 Aug 2024 00:55:42 -0500 Subject: [PATCH 105/106] Test with OpenJDK 17 (LTS) --- .github/workflows/ci.yml | 8 +++- docker/Linux-JDK17/Dockerfile | 56 +++++++++++++++++++++++++++ docker/Linux-JDK17/compose.yml | 20 ++++++++++ docker/Windows-JDK17/Dockerfile | 29 ++++++++++++++ docker/Windows-JDK17/Dockerfile-minio | 10 +++++ docker/Windows-JDK17/compose.yml | 18 +++++++++ 6 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 docker/Linux-JDK17/Dockerfile create mode 100644 docker/Linux-JDK17/compose.yml create mode 100644 docker/Windows-JDK17/Dockerfile create mode 100644 docker/Windows-JDK17/Dockerfile-minio create mode 100644 docker/Windows-JDK17/compose.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e07fd191d..a9e4c38f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest] - java: [jdk11, jdk18, graalvm] + java: [jdk11, jdk17, jdk18, graalvm] fail-fast: false steps: - name: Check out the repository @@ -19,6 +19,9 @@ jobs: - name: Test in Linux JDK 11 if: matrix.os == 'ubuntu-latest' && matrix.java == 'jdk11' run: docker compose -f docker/Linux-JDK11/docker-compose.yml up --build --exit-code-from cantaloupe + - name: Test in Linux JDK 17 (LTS) + if: matrix.os == 'ubuntu-latest' && matrix.java == 'jdk17' + run: docker compose -f docker/Linux-JDK17/compose.yml up --build --exit-code-from cantaloupe - name: Test in Linux JDK 18 if: matrix.os == 'ubuntu-latest' && matrix.java == 'jdk18' run: docker compose -f docker/Linux-JDK18/docker-compose.yml up --build --exit-code-from cantaloupe @@ -28,6 +31,9 @@ jobs: - name: Test in Windows JDK 11 if: matrix.os == 'windows-latest' && matrix.java == 'jdk11' run: docker compose -f docker/Windows-JDK11/docker-compose.yml up --build --exit-code-from cantaloupe + - name: Test in Windows JDK 17 (LTS) + if: matrix.os == 'windows-latest' && matrix.java == 'jdk17' + run: docker compose -f docker/Windows-JDK17/compose.yml up --build --exit-code-from cantaloupe - name: Test in Windows JDK 18 if: matrix.os == 'windows-latest' && matrix.java == 'jdk18' run: docker compose -f docker/Windows-JDK18/docker-compose.yml up --build --exit-code-from cantaloupe diff --git a/docker/Linux-JDK17/Dockerfile b/docker/Linux-JDK17/Dockerfile new file mode 100644 index 000000000..838170a37 --- /dev/null +++ b/docker/Linux-JDK17/Dockerfile @@ -0,0 +1,56 @@ +FROM ubuntu:lunar + +ARG DEBIAN_FRONTEND=noninteractive + +# Install various dependencies: +# * ca-certificates is needed by wget +# * ffmpeg is needed by FfmpegProcessor +# * wget download stuffs in this dockerfile +# * libopenjp2-tools is needed by OpenJpegProcessor +# * All the rest is needed by GrokProcessor +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + ffmpeg \ + wget \ + libopenjp2-tools \ + liblcms2-dev \ + libpng-dev \ + libzstd-dev \ + libtiff-dev \ + libjpeg-dev \ + zlib1g-dev \ + libwebp-dev \ + libimage-exiftool-perl \ + libgrokj2k1 \ + grokj2k-tools \ + adduser \ + openjdk-17-jdk \ + maven \ + && rm -rf /var/lib/apt/lists/* + +# Install TurboJpegProcessor dependencies +RUN mkdir -p /opt/libjpeg-turbo/lib +COPY docker/Linux-JDK11/image_files/libjpeg-turbo/lib64 /opt/libjpeg-turbo/lib + +# Install KakaduNativeProcessor dependencies +COPY dist/deps/Linux-x86-64/lib/* /usr/lib/ + +# A non-root user is needed for some FilesystemSourceTest tests to work. +ARG user=cantaloupe +ARG home=/home/$user +RUN adduser --home $home $user +RUN chown -R $user $home +USER $user +WORKDIR $home + +# Install application dependencies +COPY ./pom.xml pom.xml + +RUN echo "export JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java))))" > ~/.bashrc +RUN mvn --quiet dependency:resolve + +# Copy the code +COPY --chown=cantaloupe docker/Linux-JDK11/image_files/test.properties test.properties +COPY --chown=cantaloupe ./src src + +ENTRYPOINT mvn --batch-mode test -Pfreedeps diff --git a/docker/Linux-JDK17/compose.yml b/docker/Linux-JDK17/compose.yml new file mode 100644 index 000000000..ff2b1994b --- /dev/null +++ b/docker/Linux-JDK17/compose.yml @@ -0,0 +1,20 @@ +# +# N.B.: docker compose must be invoked from the project root directory: +# +# docker compose -f path/to/compose.yml up --exit-code-from cantaloupe +# +services: + cantaloupe: + build: + context: ../../ + dockerfile: $PWD/docker/Linux-JDK17/Dockerfile + minio: + image: minio/minio + environment: + MINIO_ACCESS_KEY: MinioUser + MINIO_SECRET_KEY: OpenSesame + hostname: minio + command: server /data + redis: + image: redis:alpine + hostname: redis diff --git a/docker/Windows-JDK17/Dockerfile b/docker/Windows-JDK17/Dockerfile new file mode 100644 index 000000000..f3726f23b --- /dev/null +++ b/docker/Windows-JDK17/Dockerfile @@ -0,0 +1,29 @@ +FROM mcr.microsoft.com/windows/servercore:ltsc2022 + +ENV chocolateyUseWindowsCompression false + +# Install the Chocolatey package manager, which makes it easier to install +# dependencies. +RUN powershell -Command \ + iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); \ + choco feature disable --name showDownloadProgress + +# Install various dependencies +# TODO: openjpeg +RUN choco install -y maven ffmpeg +RUN choco install -y openjdk --version=17.0.2 + +# Install TurboJpegProcessor dependencies TODO: libjpeg-turbo +#RUN mkdir -p /opt/libjpeg-turbo/lib +#COPY docker/Windows10-JDK11/image_files/libjpeg-turbo/lib64 c:\windows\system32 + +# Install KakaduNativeProcessor dependencies +COPY dist/deps/Windows-x86-64/lib/* c:/Windows/System32/ + +# Install application dependencies +COPY pom.xml pom.xml +RUN mvn dependency:resolve + +# Copy the code +COPY docker/Windows-JDK11/image_files/test.properties test.properties +COPY src src diff --git a/docker/Windows-JDK17/Dockerfile-minio b/docker/Windows-JDK17/Dockerfile-minio new file mode 100644 index 000000000..115389668 --- /dev/null +++ b/docker/Windows-JDK17/Dockerfile-minio @@ -0,0 +1,10 @@ +FROM mcr.microsoft.com/windows/servercore:ltsc2022 + +ENV MINIO_ACCESS_KEY=MinioUser +ENV MINIO_SECRET_KEY=OpenSesame + +RUN curl.exe --output minio.exe --url https://dl.min.io/server/minio/release/windows-amd64/minio.exe + +RUN mkdir c:\data + +CMD minio.exe server --address=:9000 c:\data diff --git a/docker/Windows-JDK17/compose.yml b/docker/Windows-JDK17/compose.yml new file mode 100644 index 000000000..17d58fa5d --- /dev/null +++ b/docker/Windows-JDK17/compose.yml @@ -0,0 +1,18 @@ +# +# N.B.: docker-compose must be invoked from the project root directory: +# +# docker compose -f path/to/compose.yml up --exit-code-from cantaloupe +# +services: + cantaloupe: + build: + context: ../../ + dockerfile: docker/Windows-JDK17/Dockerfile + minio: + build: + context: ../../ + dockerfile: docker/Windows-JDK17/Dockerfile-minio + environment: + MINIO_ACCESS_KEY: MinioUser + MINIO_SECRET_KEY: OpenSesame + hostname: minio From ecded4deea21db8442a88a0be85ecbf1cd621a71 Mon Sep 17 00:00:00 2001 From: Justin Coyne Date: Thu, 15 Aug 2024 10:19:09 -0500 Subject: [PATCH 106/106] Add JDK21 (LTS) to the test matrix --- .github/workflows/ci.yml | 8 +++- docker/Linux-JDK21/Dockerfile | 56 +++++++++++++++++++++++++++ docker/Linux-JDK21/compose.yml | 20 ++++++++++ docker/Windows-JDK21/Dockerfile | 29 ++++++++++++++ docker/Windows-JDK21/Dockerfile-minio | 10 +++++ docker/Windows-JDK21/compose.yml | 18 +++++++++ 6 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 docker/Linux-JDK21/Dockerfile create mode 100644 docker/Linux-JDK21/compose.yml create mode 100644 docker/Windows-JDK21/Dockerfile create mode 100644 docker/Windows-JDK21/Dockerfile-minio create mode 100644 docker/Windows-JDK21/compose.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9e4c38f6..5b7f08dd6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest] - java: [jdk11, jdk17, jdk18, graalvm] + java: [jdk11, jdk17, jdk18, jdk21, graalvm] fail-fast: false steps: - name: Check out the repository @@ -25,6 +25,9 @@ jobs: - name: Test in Linux JDK 18 if: matrix.os == 'ubuntu-latest' && matrix.java == 'jdk18' run: docker compose -f docker/Linux-JDK18/docker-compose.yml up --build --exit-code-from cantaloupe + - name: Test in Linux JDK 21 (LTS) + if: matrix.os == 'ubuntu-latest' && matrix.java == 'jdk21' + run: docker compose -f docker/Linux-JDK21/compose.yml up --build --exit-code-from cantaloupe - name: Test in Linux GraalVM if: matrix.os == 'ubuntu-latest' && matrix.java == 'graalvm' run: docker compose -f docker/Linux-GraalVM20/docker-compose.yml up --build --exit-code-from cantaloupe @@ -37,4 +40,7 @@ jobs: - name: Test in Windows JDK 18 if: matrix.os == 'windows-latest' && matrix.java == 'jdk18' run: docker compose -f docker/Windows-JDK18/docker-compose.yml up --build --exit-code-from cantaloupe + - name: Test in Windows JDK 21 (LTS) + if: matrix.os == 'windows-latest' && matrix.java == 'jdk21' + run: docker compose -f docker/Windows-JDK21/compose.yml up --build --exit-code-from cantaloupe # TODO: Windows+GraalVM diff --git a/docker/Linux-JDK21/Dockerfile b/docker/Linux-JDK21/Dockerfile new file mode 100644 index 000000000..78b428bee --- /dev/null +++ b/docker/Linux-JDK21/Dockerfile @@ -0,0 +1,56 @@ +FROM ubuntu:lunar + +ARG DEBIAN_FRONTEND=noninteractive + +# Install various dependencies: +# * ca-certificates is needed by wget +# * ffmpeg is needed by FfmpegProcessor +# * wget download stuffs in this dockerfile +# * libopenjp2-tools is needed by OpenJpegProcessor +# * All the rest is needed by GrokProcessor +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + ffmpeg \ + wget \ + libopenjp2-tools \ + liblcms2-dev \ + libpng-dev \ + libzstd-dev \ + libtiff-dev \ + libjpeg-dev \ + zlib1g-dev \ + libwebp-dev \ + libimage-exiftool-perl \ + libgrokj2k1 \ + grokj2k-tools \ + adduser \ + openjdk-21-jdk \ + maven \ + && rm -rf /var/lib/apt/lists/* + +# Install TurboJpegProcessor dependencies +RUN mkdir -p /opt/libjpeg-turbo/lib +COPY docker/Linux-JDK11/image_files/libjpeg-turbo/lib64 /opt/libjpeg-turbo/lib + +# Install KakaduNativeProcessor dependencies +COPY dist/deps/Linux-x86-64/lib/* /usr/lib/ + +# A non-root user is needed for some FilesystemSourceTest tests to work. +ARG user=cantaloupe +ARG home=/home/$user +RUN adduser --home $home $user +RUN chown -R $user $home +USER $user +WORKDIR $home + +# Install application dependencies +COPY ./pom.xml pom.xml + +RUN echo "export JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java))))" > ~/.bashrc +RUN mvn --quiet dependency:resolve + +# Copy the code +COPY --chown=cantaloupe docker/Linux-JDK11/image_files/test.properties test.properties +COPY --chown=cantaloupe ./src src + +ENTRYPOINT mvn --batch-mode test -Pfreedeps diff --git a/docker/Linux-JDK21/compose.yml b/docker/Linux-JDK21/compose.yml new file mode 100644 index 000000000..ff0388419 --- /dev/null +++ b/docker/Linux-JDK21/compose.yml @@ -0,0 +1,20 @@ +# +# N.B.: docker compose must be invoked from the project root directory: +# +# docker compose -f path/to/compose.yml up --exit-code-from cantaloupe +# +services: + cantaloupe: + build: + context: ../../ + dockerfile: $PWD/docker/Linux-JDK21/Dockerfile + minio: + image: minio/minio + environment: + MINIO_ACCESS_KEY: MinioUser + MINIO_SECRET_KEY: OpenSesame + hostname: minio + command: server /data + redis: + image: redis:alpine + hostname: redis diff --git a/docker/Windows-JDK21/Dockerfile b/docker/Windows-JDK21/Dockerfile new file mode 100644 index 000000000..b6741f4f3 --- /dev/null +++ b/docker/Windows-JDK21/Dockerfile @@ -0,0 +1,29 @@ +FROM mcr.microsoft.com/windows/servercore:ltsc2022 + +ENV chocolateyUseWindowsCompression false + +# Install the Chocolatey package manager, which makes it easier to install +# dependencies. +RUN powershell -Command \ + iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); \ + choco feature disable --name showDownloadProgress + +# Install various dependencies +# TODO: openjpeg +RUN choco install -y maven ffmpeg +RUN choco install -y openjdk --version=21.0.2 + +# Install TurboJpegProcessor dependencies TODO: libjpeg-turbo +#RUN mkdir -p /opt/libjpeg-turbo/lib +#COPY docker/Windows10-JDK11/image_files/libjpeg-turbo/lib64 c:\windows\system32 + +# Install KakaduNativeProcessor dependencies +COPY dist/deps/Windows-x86-64/lib/* c:/Windows/System32/ + +# Install application dependencies +COPY pom.xml pom.xml +RUN mvn dependency:resolve + +# Copy the code +COPY docker/Windows-JDK11/image_files/test.properties test.properties +COPY src src diff --git a/docker/Windows-JDK21/Dockerfile-minio b/docker/Windows-JDK21/Dockerfile-minio new file mode 100644 index 000000000..115389668 --- /dev/null +++ b/docker/Windows-JDK21/Dockerfile-minio @@ -0,0 +1,10 @@ +FROM mcr.microsoft.com/windows/servercore:ltsc2022 + +ENV MINIO_ACCESS_KEY=MinioUser +ENV MINIO_SECRET_KEY=OpenSesame + +RUN curl.exe --output minio.exe --url https://dl.min.io/server/minio/release/windows-amd64/minio.exe + +RUN mkdir c:\data + +CMD minio.exe server --address=:9000 c:\data diff --git a/docker/Windows-JDK21/compose.yml b/docker/Windows-JDK21/compose.yml new file mode 100644 index 000000000..9b286b036 --- /dev/null +++ b/docker/Windows-JDK21/compose.yml @@ -0,0 +1,18 @@ +# +# N.B.: docker-compose must be invoked from the project root directory: +# +# docker compose -f path/to/compose.yml up --exit-code-from cantaloupe +# +services: + cantaloupe: + build: + context: ../../ + dockerfile: docker/Windows-JDK21/Dockerfile + minio: + build: + context: ../../ + dockerfile: docker/Windows-JDK21/Dockerfile-minio + environment: + MINIO_ACCESS_KEY: MinioUser + MINIO_SECRET_KEY: OpenSesame + hostname: minio