Merge branch 'develop' into bugfix/auth-http-302-ids-with-slashes

.github/workflows/ci.yml

@@ -1,36 +1,46 @@
 name: CI
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - develop
+      - release/*
+  pull_request:
 jobs:
   build:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        java: [jdk11, jdk15, graalvm]
+        java: [jdk11, jdk17, jdk18, jdk21, graalvm]
       fail-fast: false
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Test in Linux JDK 11
         if: matrix.os == 'ubuntu-latest' && matrix.java == 'jdk11'
-        run: docker-compose -f docker/Linux-JDK11/docker-compose.yml up --build --exit-code-from cantaloupe
-      - name: Test in Linux JDK 15
-        if: matrix.os == 'ubuntu-latest' && matrix.java == 'jdk15'
-        run: docker-compose -f docker/Linux-JDK15/docker-compose.yml up --build --exit-code-from cantaloupe
-      - name: Test in Linux JDK 16
-        if: matrix.os == 'ubuntu-latest' && matrix.java == 'jdk16'
-        run: docker-compose -f docker/Linux-JDK16/docker-compose.yml up --build --exit-code-from cantaloupe
+        run: docker compose -f docker/Linux-JDK11/docker-compose.yml up --build --exit-code-from cantaloupe
+      - name: Test in Linux JDK 17 (LTS)
+        if: matrix.os == 'ubuntu-latest' && matrix.java == 'jdk17'
+        run: docker compose -f docker/Linux-JDK17/compose.yml up --build --exit-code-from cantaloupe
+      - name: Test in Linux JDK 18
+        if: matrix.os == 'ubuntu-latest' && matrix.java == 'jdk18'
+        run: docker compose -f docker/Linux-JDK18/docker-compose.yml up --build --exit-code-from cantaloupe
+      - name: Test in Linux JDK 21 (LTS)
+        if: matrix.os == 'ubuntu-latest' && matrix.java == 'jdk21'
+        run: docker compose -f docker/Linux-JDK21/compose.yml up --build --exit-code-from cantaloupe
       - name: Test in Linux GraalVM
         if: matrix.os == 'ubuntu-latest' && matrix.java == 'graalvm'
-        run: docker-compose -f docker/Linux-GraalVM20/docker-compose.yml up --build --exit-code-from cantaloupe
+        run: docker compose -f docker/Linux-GraalVM20/docker-compose.yml up --build --exit-code-from cantaloupe
       - name: Test in Windows JDK 11
         if: matrix.os == 'windows-latest' && matrix.java == 'jdk11'
-        run: docker-compose -f docker/Windows-JDK11/docker-compose.yml up --build --exit-code-from cantaloupe
-      - name: Test in Windows JDK 15
-        if: matrix.os == 'windows-latest' && matrix.java == 'jdk15'
-        run: docker-compose -f docker/Windows-JDK15/docker-compose.yml up --build --exit-code-from cantaloupe
-      - name: Test in Windows JDK 16
-        if: matrix.os == 'windows-latest' && matrix.java == 'jdk16'
-        run: docker-compose -f docker/Windows-JDK16/docker-compose.yml up --build --exit-code-from cantaloupe
+        run: docker compose -f docker/Windows-JDK11/docker-compose.yml up --build --exit-code-from cantaloupe
+      - name: Test in Windows JDK 17 (LTS)
+        if: matrix.os == 'windows-latest' && matrix.java == 'jdk17'
+        run: docker compose -f docker/Windows-JDK17/compose.yml up --build --exit-code-from cantaloupe
+      - name: Test in Windows JDK 18
+        if: matrix.os == 'windows-latest' && matrix.java == 'jdk18'
+        run: docker compose -f docker/Windows-JDK18/docker-compose.yml up --build --exit-code-from cantaloupe
+      - name: Test in Windows JDK 21 (LTS)
+        if: matrix.os == 'windows-latest' && matrix.java == 'jdk21'
+        run: docker compose -f docker/Windows-JDK21/compose.yml up --build --exit-code-from cantaloupe
       # TODO: Windows+GraalVM
-

.gitignore

@@ -8,7 +8,7 @@ Thumbs.db
 .settings/*
 
 # IntelliJ stuff
-/.idea/*
+/.idea
 /*.iml
 /*.ipr
 /*.iws

CHANGES.md

@@ -1,12 +1,50 @@
 # Change Log
 
+## 6.0
+
+### Endpoints
+
+* Image and information responses include a `Last-Modified` header when
+  possible.
+* The health endpoint is enabled via `endpoint.health.enabled` rather than
+  `endpoint.api.enabled`.
+* Added an HTTP API method to purge all infos from the derivative cache.
+* Added a configuration option to automatically purge source-cached images
+  whose format cannot be inferred.
+
+### Sources
+
+* HttpSource supports a client HTTP proxy. (Thanks to @mightymax and
+  @mlindeman)
+* HttpSource can be configured to send a ranged GET request instead of a HEAD
+  request, enabling it to work with pre-signed URLs that do not allow HEAD
+  requests.
+* S3Source supports multiple endpoints when using ScriptLookupStrategy.
+
+### Caches
+
+* S3Cache uses multipart uploads, which reduces memory usage when caching
+  derivatives larger than 5 MB.
+
+### Delegate Script
+
+* The delegate script pathname can be set using the
+  `-Dcantaloupe.delegate_script` VM argument, which takes precedence over the
+  `delegate_script.pathname` configuration key.
+* The delegate script's `metadata` context key contains a new field,
+  `xmp_elements`, that provides a high-level key-value view of the XMP data.
+
 ## 5.0.6
 
 * IIIF information endpoints always return JSON in HTTP 4xx responses.
+* Fixed a bug whereby the values of the `operations` and `page_count` keys
+  in the delegate context were not set.
 * TurboJpegProcessor is able to generate non-JPEG derivative images, which
   fixes an HTTP 415 error that would occur when trying to do that.
 * Fixed a crop-offset bug that could occur when using PdfBoxProcessor to
   generate JPEGs with libjpeg-turbo active.
+* Updating libraries to fix security issues. Full details in [#634](https://github.com/cantaloupe-project/cantaloupe/issues/634)  
+* Update of Jena to 4.8 requires RDF to have a populated rdf:about field. May impact some XMP header processing.
 
 ## 5.0.5
 

CREDITS.md

@@ -0,0 +1,9 @@
+# Credits
+
+The success of this project is down to the great work done by the developers at the University of Illinois and particularly Alex Dolski who has been the main developer on the project. In 2023 the IIIF consortium offered to help release a security patch as so many IIIF community members are using this great software. 
+
+- Cantaloupe 5.0.6
+ * [Glen Robson](https://github.com/glenrobson/) with support from the [IIIF Consortium](https://iiif.io/community/consortium/)
+
+- Cantaloupe v1 - v6
+ * [Alex Dolski](https://github.com/adolski), University of Illinois

README.md

@@ -12,7 +12,8 @@
 
 ### Command line
 
-* `mvn clean compile exec:java -Dcantaloupe.config=...` will build and run the
+* `cp cantaloupe.properties.sample cantaloupe.properties` then edit cantaloupe.properties per your needs.
+* `mvn clean compile exec:java -Dcantaloupe.config=cantaloupe.properties` will build and run the
   project using the embedded web server listening on the port(s) specified in
   `cantaloupe.properties`.
 * `mvn clean package -DskipTests` will build a release JAR in the `target`
@@ -25,7 +26,7 @@
 1. Add a new run configuration using the "Java Application" template or
    similar.
 2. Set the main class to `edu.illinois.library.cantaloupe.StandaloneEntry` and
-   add the `-Dcantaloupe.config=...` VM option.
+   add the `-Dcantaloupe.config=cantaloupe.properties` VM option.
 
 ## Test
 

UPGRADING.md

@@ -3,6 +3,20 @@
 If you are skipping versions, work through these sections backwards from your
 current version.
 
+## 5.0 → 6.0
+
+1. Add the following keys from the sample configuration:
+    * `endpoint.health.enabled`
+    * `HttpSource.proxy.http.host`
+    * `HttpSource.proxy.http.port`
+    * `HttpSource.BasicLookupStrategy.send_head_requests`
+    * `processor.purge_incompatible_from_source_cache`
+2. Add the following methods from the sample delegate script:
+    * `jdbcsource_last_modified()`
+3. If you are using a Java delegate, add the following method to your delegate 
+   class: 
+    * `getJDBCSourceLastModified()`
+
 ## 4.1.x → 5.0
 
 1.  Note that the application is now packaged as a JAR file which can no longer

cantaloupe.properties.sample

@@ -88,6 +88,8 @@ delegate_script.enabled = false
 # !! This can be an absolute path, or a filename; if only a filename is
 # specified, it will be searched for in the same folder as this file, and
 # then the current working directory.
+# The delegate script pathname can also be set using the
+# -Dcantaloupe.delegate_script VM argument, which overrides this value.
 delegate_script.pathname = delegates.rb
 
 ###########################################################################
@@ -126,6 +128,9 @@ endpoint.api.enabled = false
 endpoint.api.username =
 endpoint.api.secret =
 
+# Enables the health check endpoint, at /health.
+endpoint.health.enabled = true
+
 # If true, sources and caches will be checked, resulting in a more robust
 # but slower health check. Set this to false if these services already have
 # their own health checks.
@@ -170,6 +175,10 @@ HttpSource.allow_insecure = false
 # Request timeout in seconds.
 HttpSource.request_timeout =
 
+# !! Client HTTP proxy.
+HttpSource.proxy.http.host =
+HttpSource.proxy.http.port =
+
 # Tells HttpSource how to look up resources. Allowed values are
 # `BasicLookupStrategy` and `ScriptLookupStrategy`. ScriptLookupStrategy
 # uses a delegate method for dynamic lookups; see the user manual.
@@ -187,6 +196,12 @@ HttpSource.BasicLookupStrategy.url_suffix =
 HttpSource.BasicLookupStrategy.auth.basic.username =
 HttpSource.BasicLookupStrategy.auth.basic.secret =
 
+# Before an image is retrieved, a preliminary request is sent to check
+# various characteristics. Typically this is a HEAD request, but some
+# resources, such as those using pre-signed URLs, may not support HEAD
+# requests. This key enables a ranged GET request to be sent instead.
+HttpSource.BasicLookupStrategy.send_head_requests = true
+
 # Read data in chunks when it may be more efficient. (This also may end up
 # being less efficient, depending on many variables; see the user manual.)
 HttpSource.chunking.enabled = true
@@ -359,6 +374,10 @@ processor.stream_retrieval_strategy = StreamStrategy
 # * `AbortStrategy` causes the request to fail.
 processor.fallback_retrieval_strategy = DownloadStrategy
 
+# If true, images stored in the source cache for which no format can be
+# inferred will be purged.
+processor.purge_incompatible_from_source_cache = false
+
 # Resolution of vector rasterization (of e.g. PDFs) at a scale of 1.
 processor.dpi = 150
 

delegates.rb.sample

@@ -221,10 +221,17 @@ class CustomDelegate
   #
   # 1. String URI
   # 2. Hash with the following keys:
-  #     * `uri`      [String] (required)
-  #     * `username` [String] For HTTP Basic authentication (optional).
-  #     * `secret`   [String] For HTTP Basic authentication (optional).
-  #     * `headers`  [Hash<String,String>] Hash of request headers (optional).
+  #     * `uri`               [String] (required)
+  #     * `username`          [String] For HTTP Basic authentication
+  #                           (optional).
+  #     * `secret`            [String] For HTTP Basic authentication
+  #                           (optional).
+  #     * `headers`           [Hash<String,String>] Hash of request headers
+  #                           (optional).
+  #     * `send_head_request` [Boolean] Optional; defaults to `true`. See the
+  #                           documentation of the
+  #                           `HttpSource.BasicLookupStrategy.send_head_requests`
+  #                           configuration key.
   # 3. nil if not found.
   #
   # N.B.: this method should not try to perform authorization. `authorize()`
@@ -241,18 +248,33 @@ class CustomDelegate
   # should be used instead.
   #
   # @param options [Hash] Empty hash.
-  # @return [String] Identifier of the image corresponding to the given
-  #                  identifier in the database.
+  # @return [String, nil] Database identifier of the image corresponding to the
+  #                       identifier in the context, or nil if not found.
   #
   def jdbcsource_database_identifier(options = {})
   end
 
+  ##
+  # Returns either the last-modified timestamp of an image in ISO 8601 format,
+  # or an SQL statement that can be used to retrieve it from a `TIMESTAMP`-type
+  # column in the database. In the latter case, the "SELECT" and "FROM" clauses
+  # should be in uppercase in order to be autodetected.
+  #
+  # Implementing this method is optional, but may be necessary for certain
+  # features (like `Last-Modified` response headers) to work.
+  #
+  # @param options [Hash] Empty hash.
+  # @return [String, nil]
+  #
+  def jdbcsource_last_modified(options = {})
+  end
+
   ##
   # Returns either the media (MIME) type of an image, or an SQL statement that
-  # can be used to retrieve it, if it is stored in the database. In the latter
-  # case, the "SELECT" and "FROM" clauses should be in uppercase in order to
-  # be autodetected. If nil is returned, the media type will be inferred some
-  # other way, such as by identifier extension or magic bytes.
+  # can be used to retrieve it from a `CHAR`-type column in the database. In
+  # the latter case, the "SELECT" and "FROM" clauses should be in uppercase in
+  # order to be autodetected. If nil is returned, the media type will be
+  # inferred some other way, such as by identifier extension or magic bytes.
   #
   # @param options [Hash] Empty hash.
   # @return [String, nil]
@@ -273,8 +295,11 @@ class CustomDelegate
   # should be used instead.
   #
   # @param options [Hash] Empty hash.
-  # @return [Hash<String,Object>,nil] Hash containing `bucket` and `key` keys;
-  #                                   or nil if not found.
+  # @return [Hash<String,Object>,nil] Hash containing `bucket` and `key` keys.
+  #         It may also contain an `endpoint` key, indicating that the endpoint
+  #         is different from the one set in the configuration. In that case,
+  #         it may also contain `region`, `access_key_id`, and/or
+  #         `secret_access_key` keys.
   #
   def s3source_object_info(options = {})
   end
@@ -356,7 +381,14 @@ class CustomDelegate
   #         "Field2Name": value
   #     ],
   #     "xmp_string": "<rdf:RDF>...</rdf:RDF>",
-  #     "xmp_model": https://jena.apache.org/documentation/javadoc/jena/org/apache/jena/rdf/model/Model.html
+  #     "xmp_model": See https://jena.apache.org/documentation/javadoc/jena/org/apache/jena/rdf/model/Model.html,
+  #     "xmp_elements": {
+  #         "Field1Name": "value",
+  #         "Field2Name": [
+  #             "value1",
+  #             "value2"
+  #         ]
+  #     },
   #     "native": {
   #         # structure varies
   #     }
@@ -366,10 +398,13 @@ class CustomDelegate
   # * The `exif` key refers to embedded EXIF data. This also includes IFD0
   #   metadata from source TIFFs, whether or not an EXIF IFD is present.
   # * The `iptc` key refers to embedded IPTC IIM data.
-  # * The `xmp_string` key refers to raw embedded XMP data, which may or may
-  #   not contain EXIF and/or IPTC information.
+  # * The `xmp_string` key refers to raw embedded XMP data.
   # * The `xmp_model` key contains a Jena Model object pre-loaded with the
   #   contents of `xmp_string`.
+  # * The `xmp_elements` key contains a view of the embedded XMP data as key-
+  #   value pairs. This is convenient to use, but may not work correctly with
+  #   all XMP fields--in particular, those that cannot be expressed as
+  #   key-value pairs.
   # * The `native` key refers to format-specific metadata.
   #
   # Any combination of the above keys may be present or missing depending on

docker/Linux-GraalVM20/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:latest
+FROM ubuntu:lunar
 
 ENV JAVA_HOME=/opt/graalvm-ce-java11-20.3.0
 ENV GRAALVM_HOME=/opt/graalvm-ce-java11-20.3.0
@@ -20,6 +20,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
                 zlib1g-dev \
                 libwebp-dev \
                 libimage-exiftool-perl \
+                libgrokj2k1 \
+                grokj2k-tools \ 
+                adduser \
     && rm -rf /var/lib/apt/lists/*
 
 # Install TurboJpegProcessor dependencies
@@ -30,10 +33,10 @@ COPY docker/Linux-JDK11/image_files/libjpeg-turbo/lib64 /opt/libjpeg-turbo/lib
 COPY dist/deps/Linux-x86-64/lib/* /usr/lib/
 
 # Install GrokProcessor dependencies
-RUN wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/libgrokj2k1_7.6.5-1_amd64.deb \
-    && wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/grokj2k-tools_7.6.5-1_amd64.deb \
-    && dpkg -i ./libgrokj2k1_7.6.5-1_amd64.deb \
-    && dpkg -i --ignore-depends=libjpeg62-turbo ./grokj2k-tools_7.6.5-1_amd64.deb
+#RUN wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/libgrokj2k1_7.6.5-1_amd64.deb \
+#    && wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/grokj2k-tools_7.6.5-1_amd64.deb \
+#    && dpkg -i --ignore-depends=libjpeg62-turbo ./grokj2k-tools_7.6.5-1_amd64.deb
+#    && dpkg -i ./libgrokj2k1_7.6.5-1_amd64.deb \
 
 # Install GraalVM
 RUN wget -q https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-20.3.0/graalvm-ce-java11-linux-amd64-20.3.0.tar.gz \