From b3ec0c2fe46b0fe7f5e5799d4c090efa4c068e40 Mon Sep 17 00:00:00 2001 From: Justin Coyne Date: Wed, 31 Jul 2024 10:36:41 -0500 Subject: [PATCH] Remove Grok --- README.md | 1 - cantaloupe.properties.sample | 10 +- docker/Linux-GraalVM20/Dockerfile | 8 - docker/Linux-JDK11/Dockerfile | 8 - docker/Linux-JDK18/Dockerfile | 9 - pom.xml | 3 +- .../library/cantaloupe/config/Key.java | 1 - .../processor/AutomaticSelectionStrategy.java | 3 +- .../cantaloupe/processor/GrokProcessor.java | 613 ------------------ .../processor/ProcessorFactory.java | 1 - src/main/resources/admin.vm | 21 +- .../processor/GrokProcessorPerformance.java | 72 -- .../AutomaticSelectionStrategyTest.java | 3 +- .../processor/GrokProcessorTest.java | 111 ---- .../resource/admin/AdminResourceUITest.java | 6 - 15 files changed, 5 insertions(+), 865 deletions(-) delete mode 100644 src/main/java/edu/illinois/library/cantaloupe/processor/GrokProcessor.java delete mode 100644 src/test/java/edu/illinois/library/cantaloupe/perf/processor/GrokProcessorPerformance.java delete mode 100644 src/test/java/edu/illinois/library/cantaloupe/processor/GrokProcessorTest.java diff --git a/README.md b/README.md index 3b53f62d9..1316b4940 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,6 @@ continuous integration. The following dependencies are required: * MinIO (for S3SourceTest & S3CacheTest) * FFmpeg (for FfmpegProcessorTest) -* Grok (for GrokProcessorTest) * OpenJPEG (for OpenJpegProcessorTest) * TurboJPEG with Java binding (for TurboJpegProcessorTest) * Redis (for RedisCacheTest) diff --git a/cantaloupe.properties.sample b/cantaloupe.properties.sample index e205ad760..9f8087752 100644 --- a/cantaloupe.properties.sample +++ b/cantaloupe.properties.sample @@ -331,7 +331,7 @@ JdbcSource.connection_timeout = 10 processor.selection_strategy = AutomaticSelectionStrategy # Built-in processors are `Java2dProcessor`, TurboJpegProcessor`, -# `KakaduNativeProcessor`, `OpenJpegProcessor`, `GrokProcessor`,`JaiProcessor`, +# `KakaduNativeProcessor`, `OpenJpegProcessor`, `JaiProcessor`, # `PdfBoxProcessor`, and `FfmpegProcessor`. # Some of these have third-party dependencies and won't work out-of-the-box. @@ -443,14 +443,6 @@ FfmpegProcessor.path_to_binaries = # Overrides the PATH. OpenJpegProcessor.path_to_binaries = -#---------------------------------------- -# GrokProcessor -#---------------------------------------- - -# Optional absolute path of the directory containing grk_decompress. -# Overrides the PATH. -GrokProcessor.path_to_binaries = - #---------------------------------------- # PdfBoxProcessor #---------------------------------------- diff --git a/docker/Linux-GraalVM20/Dockerfile b/docker/Linux-GraalVM20/Dockerfile index 234616e03..ca40d7174 100644 --- a/docker/Linux-GraalVM20/Dockerfile +++ b/docker/Linux-GraalVM20/Dockerfile @@ -20,8 +20,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ zlib1g-dev \ libwebp-dev \ libimage-exiftool-perl \ - libgrokj2k1 \ - grokj2k-tools \ adduser \ && rm -rf /var/lib/apt/lists/* @@ -32,12 +30,6 @@ COPY docker/image_files/libjpeg-turbo/lib64 /opt/libjpeg-turbo/lib # Install KakaduNativeProcessor dependencies COPY dist/deps/Linux-x86-64/lib/* /usr/lib/ -# Install GrokProcessor dependencies -#RUN wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/libgrokj2k1_7.6.5-1_amd64.deb \ -# && wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/grokj2k-tools_7.6.5-1_amd64.deb \ -# && dpkg -i --ignore-depends=libjpeg62-turbo ./grokj2k-tools_7.6.5-1_amd64.deb -# && dpkg -i ./libgrokj2k1_7.6.5-1_amd64.deb \ - # Install GraalVM RUN wget -q https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-20.3.0/graalvm-ce-java11-linux-amd64-20.3.0.tar.gz \ && tar xfz graalvm-ce-java11-linux-amd64-20.3.0.tar.gz \ diff --git a/docker/Linux-JDK11/Dockerfile b/docker/Linux-JDK11/Dockerfile index acadf2c20..2ad6ebfb2 100644 --- a/docker/Linux-JDK11/Dockerfile +++ b/docker/Linux-JDK11/Dockerfile @@ -18,8 +18,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ zlib1g-dev \ libwebp-dev \ libimage-exiftool-perl \ - libgrokj2k1 \ - grokj2k-tools \ adduser \ && rm -rf /var/lib/apt/lists/* @@ -30,12 +28,6 @@ COPY docker/image_files/libjpeg-turbo/lib64 /opt/libjpeg-turbo/lib # Install KakaduNativeProcessor dependencies COPY dist/deps/Linux-x86-64/lib/* /usr/lib/ -# Install GrokProcessor dependencies -#RUN wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/libgrokj2k1_7.6.5-1_amd64.deb \ -# && wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/grokj2k-tools_7.6.5-1_amd64.deb \ -# && dpkg -i ./libgrokj2k1_7.6.5-1_amd64.deb \ -# && dpkg -i --ignore-depends=libjpeg62-turbo ./grokj2k-tools_7.6.5-1_amd64.deb - # A non-root user is needed for some FilesystemSourceTest tests to work. ARG user=cantaloupe ARG home=/home/$user diff --git a/docker/Linux-JDK18/Dockerfile b/docker/Linux-JDK18/Dockerfile index d7f545d85..971466b5c 100644 --- a/docker/Linux-JDK18/Dockerfile +++ b/docker/Linux-JDK18/Dockerfile @@ -9,7 +9,6 @@ ARG DEBIAN_FRONTEND=noninteractive # * ffmpeg is needed by FfmpegProcessor # * wget download stuffs in this dockerfile # * libopenjp2-tools is needed by OpenJpegProcessor -# * All the rest is needed by GrokProcessor RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ ffmpeg \ @@ -23,8 +22,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ zlib1g-dev \ libwebp-dev \ libimage-exiftool-perl \ - libgrokj2k1 \ - grokj2k-tools \ adduser \ && rm -rf /var/lib/apt/lists/* @@ -35,12 +32,6 @@ COPY docker/image_files/libjpeg-turbo/lib64 /opt/libjpeg-turbo/lib # Install KakaduNativeProcessor dependencies COPY dist/deps/Linux-x86-64/lib/* /usr/lib/ -# Install various other dependencies that aren't in apt -# Install GrokProcessor dependencies -#RUN wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/libgrokj2k1_7.6.5-1_amd64.deb \ -# && wget -q https://github.com/GrokImageCompression/grok/releases/download/v7.6.5/grokj2k-tools_7.6.5-1_amd64.deb \ -# && dpkg -i ./libgrokj2k1_7.6.5-1_amd64.deb \ -# && dpkg -i --ignore-depends=libjpeg62-turbo ./grokj2k-tools_7.6.5-1_amd64.deb \ # Install OpenJDK RUN wget -q https://download.java.net/java/GA/jdk18/43f95e8614114aeaa8e8a5fcf20a682d/36/GPL/openjdk-18_linux-x64_bin.tar.gz \ && tar xfz openjdk-18_linux-x64_bin.tar.gz \ diff --git a/pom.xml b/pom.xml index bc7f4803a..a1c939d35 100644 --- a/pom.xml +++ b/pom.xml @@ -340,7 +340,7 @@ org.seleniumhq.selenium htmlunit-driver - 2.21 + 2.21 test @@ -568,7 +568,6 @@ FfmpegProcessorTest *Kakadu*Test OpenJpegProcessorTest - GrokProcessorTest RedisCacheTest TurboJpegProcessorTest TurboJPEG*Test diff --git a/src/main/java/edu/illinois/library/cantaloupe/config/Key.java b/src/main/java/edu/illinois/library/cantaloupe/config/Key.java index 2ac19ce09..f61d07387 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/config/Key.java +++ b/src/main/java/edu/illinois/library/cantaloupe/config/Key.java @@ -144,7 +144,6 @@ public enum Key { MAX_SCALE("max_scale"), META_IDENTIFIER_TRANSFORMER("meta_identifier.transformer"), OPENJPEGPROCESSOR_PATH_TO_BINARIES("OpenJpegProcessor.path_to_binaries"), - GROKPROCESSOR_PATH_TO_BINARIES("GrokProcessor.path_to_binaries"), OVERLAY_ENABLED("overlays.BasicStrategy.enabled"), OVERLAY_IMAGE("overlays.BasicStrategy.image"), OVERLAY_INSET("overlays.BasicStrategy.inset"), diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/AutomaticSelectionStrategy.java b/src/main/java/edu/illinois/library/cantaloupe/processor/AutomaticSelectionStrategy.java index 83983d20e..9a1b2c9f7 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/AutomaticSelectionStrategy.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/AutomaticSelectionStrategy.java @@ -12,8 +12,7 @@ class AutomaticSelectionStrategy implements SelectionStrategy { private static final List> JP2_CANDIDATES = List.of( KakaduNativeProcessor.class, - OpenJpegProcessor.class, - GrokProcessor.class); + OpenJpegProcessor.class); private static final List> JPG_CANDIDATES = List.of( TurboJpegProcessor.class, Java2dProcessor.class); diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/GrokProcessor.java b/src/main/java/edu/illinois/library/cantaloupe/processor/GrokProcessor.java deleted file mode 100644 index 1aa85fb97..000000000 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/GrokProcessor.java +++ /dev/null @@ -1,613 +0,0 @@ -package edu.illinois.library.cantaloupe.processor; - -import edu.illinois.library.cantaloupe.Application; -import edu.illinois.library.cantaloupe.async.TaskQueue; -import edu.illinois.library.cantaloupe.async.ThreadPool; -import edu.illinois.library.cantaloupe.config.Configuration; -import edu.illinois.library.cantaloupe.config.Key; -import edu.illinois.library.cantaloupe.image.Dimension; -import edu.illinois.library.cantaloupe.image.Format; -import edu.illinois.library.cantaloupe.image.Info; -import edu.illinois.library.cantaloupe.image.Metadata; -import edu.illinois.library.cantaloupe.image.Rectangle; -import edu.illinois.library.cantaloupe.operation.Encode; -import edu.illinois.library.cantaloupe.operation.Operation; -import edu.illinois.library.cantaloupe.operation.OperationList; -import edu.illinois.library.cantaloupe.operation.ReductionFactor; -import edu.illinois.library.cantaloupe.operation.Scale; -import edu.illinois.library.cantaloupe.operation.Crop; -import edu.illinois.library.cantaloupe.processor.codec.ImageReader; -import edu.illinois.library.cantaloupe.processor.codec.ImageReaderFactory; -import edu.illinois.library.cantaloupe.processor.codec.ImageWriterFactory; -import edu.illinois.library.cantaloupe.processor.codec.ImageWriterFacade; -import edu.illinois.library.cantaloupe.processor.codec.jpeg2000.JPEG2000MetadataReader; -import edu.illinois.library.cantaloupe.processor.codec.ReaderHint; -import edu.illinois.library.cantaloupe.source.stream.BufferedImageInputStream; -import edu.illinois.library.cantaloupe.util.CommandLocator; -import org.apache.commons.lang3.SystemUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.imageio.stream.FileImageInputStream; -import java.awt.image.BufferedImage; -import java.io.BufferedInputStream; -import java.io.ByteArrayOutputStream; -import java.io.EOFException; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.FileTime; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collections; -import java.util.EnumSet; -import java.util.List; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - *

Processor using the Grok {@literal grk_decompress} command-line - * tool.

- * - *

{@literal grk_decompress} is used for cropping and acquiring a scale- - * reduced intermediate BMP, which is streamed into an Image I/O reader. (BMP - * does not copy embedded ICC profiles into output images, but {@literal - * grk_decompress} converts the RGB source data itself.) Java 2D is used for - * all remaining processing steps.

- * - *

{@literal grk_decompress} reads and writes the files named in the - * {@literal -i} and {@literal -o} arguments passed to it, respectively. The - * file in the {@literal -o} argument must have a {@literal .bmp} extension. - * This means that it's not possible to natively write to a {@link - * Process#getInputStream() process input stream}. The way this is dealt with - * differs between Windows and Unix:

- * - *
- *
Unix
- *
A symlink is created by {@link #initialize()} from {@literal - * /tmp/whatever.bmp} to {@literal /dev/stdout}, and set to {@link - * java.io.File#deleteOnExit() delete on exit}. {@literal grk_decompress} - * then effectively writes to standard output, which can be read from a - * {@link Process#getInputStream() process' input stream}.
- *
Windows
- *
Windows doesn't have anything like {@literal /dev/stdout}. (Actually - * it has {@literal CON}, but experimentation reveals it won't work.) So - * {@literal grk_decompress} instead writes to a temporary file which is - * deleted after being read. This is slower than the Unix technique.
- *
- * - *

Although {@literal grk_decompress} is used for reading images, - * {@literal grk_dump} is not used for reading metadata. - * {@link JPEG2000MetadataReader} is used instead, as this doesn't require - * invoking a process.

- * - *

grk_decompress version 7.7.0 is recommended.

- * - * @author Alex Dolski UIUC - * @author Aaron Boxer - */ -class GrokProcessor extends AbstractProcessor implements FileProcessor { - - private static final Logger LOGGER = - LoggerFactory.getLogger(GrokProcessor.class); - - /** - * Number of decomposition levels assumed to be contained in the image when - * that information cannot be obtained for some reason. 5 is the default - * used by most JP2 encoders. Setting this to a value higher than that could - * cause decoding errors, and setting it to lower could have a performance - * cost. - */ - private static final short FALLBACK_NUM_DWT_LEVELS = 5; - - private static final String GRK_DECOMPRESS_NAME = "grk_decompress"; - - /** - * Used only in Windows. - */ - private static final String WINDOWS_SCRATCH_DIR_NAME = - GrokProcessor.class.getSimpleName() + "-scratch"; - - /** - * Set by {@link #initialize()}. - */ - private static final AtomicBoolean IS_INITIALIZATION_ATTEMPTED = - new AtomicBoolean(false); - - /** - * Set by {@link #initialize()}. - */ - private static String initializationError; - - /** - * @see - * OpenJpegProcessor operating on low bit-depth images - */ - private final static Format intermediateFormat = Format.get("bmp"); - - private Path sourceFile; - - /** - * {@literal grk_decompress} has problems with some files that are missing - * a JP2 filename extension. When {@link #sourceFile} does not contain one, - * this will be set by {@link #setSourceFile(Path)} and used instead of - * {@link #sourceFile}. - * - * N.B.: the symlink must be cleaned up when processing is complete. - */ - private Path sourceSymlink; - - private static String getPath() { - String searchPath = Configuration.getInstance(). - getString(Key.GROKPROCESSOR_PATH_TO_BINARIES); - return CommandLocator.locate(GRK_DECOMPRESS_NAME, searchPath); - } - - /** - * Used only in Windows. - * - * @return Thread-safe path of an intermediate image from {@literal - * grk_decompress} based on the given operation list. - */ - private static Path getIntermediateImageFile(OperationList opList) { - final String name = opList.toFilename() + "-" + - Thread.currentThread().getName() + "." + - intermediateFormat.getPreferredExtension(); - return getScratchDir().resolve(name); - } - - /** - * Used only in Windows. - * - * @return Path to the scratch directory that stores output images from - * {@literal grk_decompress}. - */ - private static Path getScratchDir() { - return Application.getTempPath().resolve(WINDOWS_SCRATCH_DIR_NAME); - } - - private static synchronized void initialize() { - IS_INITIALIZATION_ATTEMPTED.set(true); - - try { - // Check for the presence of grk_decompress. - invoke(); - - if (isWindows()) { - initializeForWindows(); - } - // Unix doesn't need any initialization. - } catch (IOException e) { - initializationError = e.getMessage(); - } - } - - private static boolean isWindows() { - return SystemUtils.IS_OS_WINDOWS; - } - - private static void initializeForWindows() throws IOException { - final Path scratchDir = getScratchDir(); - - if (!Files.exists(scratchDir)) { - LOGGER.debug("Creating {}", scratchDir); - Files.createDirectories(scratchDir); - } - } - - private static void invoke() throws IOException { - final ProcessBuilder pb = new ProcessBuilder(); - List command = new ArrayList<>(); - command.add(getPath()); - pb.command(command); - String commandString = String.join(" ", pb.command()); - LOGGER.info("invoke(): {}", commandString); - pb.start(); - } - - - /** - * For testing only! - */ - static synchronized void resetInitialization() { - IS_INITIALIZATION_ATTEMPTED.set(false); - initializationError = null; - } - - - private static String toString(ByteArrayOutputStream os) { - return new String(os.toByteArray(), StandardCharsets.UTF_8); - } - - GrokProcessor() { - if (!IS_INITIALIZATION_ATTEMPTED.get()) { - initialize(); - } - } - - @Override - public void close() { - if (sourceSymlink != null) { - TaskQueue.getInstance().submit(() -> { - LOGGER.trace("Deleting {}", sourceSymlink); - Files.deleteIfExists(sourceSymlink); - return null; - }); - } - } - - /** - *

Creates a symlink to {@literal /dev/stdout} in a temporary directory. - * The symlink is for the exclusive use of the instance and should be - * cleaned up when no longer needed.

- * - *

Not used in Windows.

- */ - private Path createStdoutSymlink() throws IOException { - final String name = GrokProcessor.class.getSimpleName() + "-" + - UUID.randomUUID() + "." + - intermediateFormat.getPreferredExtension(); - final Path link = Application.getTempPath().resolve(name); - final Path devStdout = Paths.get("/dev/stdout"); - - LOGGER.debug("Creating link from {} to {}", link, devStdout); - return Files.createSymbolicLink(link, devStdout); - } - - @Override - public Set getAvailableOutputFormats() { - final Set outputFormats; - if (Format.get("jp2").equals(getSourceFormat())) { - outputFormats = ImageWriterFactory.supportedFormats(); - } else { - outputFormats = Collections.unmodifiableSet(Collections.emptySet()); - } - return outputFormats; - } - - @Override - public String getInitializationError() { - if (!IS_INITIALIZATION_ATTEMPTED.get()) { - initialize(); - } - return initializationError; - } - - @Override - public Path getSourceFile() { - return sourceFile; - } - - @Override - public void setSourceFile(Path sourceFile) { - this.sourceFile = sourceFile; - - // N.B.: As of version 2.3.0, grk_decompress fails to open certain - // files without a .jp2 extension. This is most notably an issue when - // reading from the source cache, as those files don't have extensions. - // - // Our workaround, when we are reading a file without a recognized - // extension, is to create a symlink to the file to read that has the - // extension grk_decompress needs, remembering to delete it in close(). - // - // We still must "touch" the source file, so that FilesystemCache knows - // it's been accessed. - final String filename = sourceFile.toString().toLowerCase(); - if (!filename.endsWith(".jp2") && !filename.endsWith(".jpx") && - !filename.endsWith(".j2k")) { - // Touch the file (in the background since we don't care about - // the result). - TaskQueue.getInstance().submit(() -> { - try { - Files.setLastModifiedTime(sourceFile, - FileTime.from(Instant.now())); - } catch (IOException e) { - LOGGER.error("setSourceFile(): failed to touch file: {}", - e.getMessage()); - } - }); - - // Create the symlink. - try { - final String name = GrokProcessor.class.getSimpleName() + - "-" + UUID.randomUUID() + ".jp2"; - sourceSymlink = Application.getTempPath().resolve(name); - - LOGGER.trace("Creating link from {} to {}", - sourceSymlink, sourceFile); - Files.createSymbolicLink(sourceSymlink, sourceFile); - } catch (IOException e) { - LOGGER.error("setSourceFile(): {}", e.getMessage()); - sourceSymlink = null; - } - } - } - - @Override - public boolean supportsSourceFormat(Format format) { - return Format.get("jp2").equals(format); - } - - @Override - public Info readInfo() throws IOException { - final Info info = new Info(); - info.setSourceFormat(getSourceFormat()); - - try (final JPEG2000MetadataReader reader = new JPEG2000MetadataReader()) { - reader.setSource(new BufferedImageInputStream( - new FileImageInputStream(getSourceFile().toFile()))); - - final Metadata metadata = new Metadata(); - byte[] bytes = reader.getEXIF(); - if (bytes != null) { - try (edu.illinois.library.cantaloupe.image.exif.Reader exifReader = - new edu.illinois.library.cantaloupe.image.exif.Reader()) { - exifReader.setSource(bytes); - metadata.setEXIF(exifReader.read()); - } - } - bytes = reader.getIPTC(); - if (bytes != null) { - try (edu.illinois.library.cantaloupe.image.iptc.Reader iptcReader = - new edu.illinois.library.cantaloupe.image.iptc.Reader()) { - iptcReader.setSource(bytes); - metadata.setIPTC(iptcReader.read()); - } - } - metadata.setXMP(reader.getXMP()); - info.setMetadata(metadata); - info.setNumResolutions(reader.getNumDecompositionLevels() + 1); - - Info.Image image = info.getImages().get(0); - image.setSize(new Dimension(reader.getWidth(), reader.getHeight())); - image.setTileSize(new Dimension(reader.getTileWidth(), reader.getTileHeight())); - // JP2 tile dimensions are inverted, so swap them - if ((image.width > image.height && image.tileWidth < image.tileHeight) || - (image.width < image.height && image.tileWidth > image.tileHeight)) { - int tmp = image.tileWidth; - //noinspection SuspiciousNameCombination - image.tileWidth = image.tileHeight; - image.tileHeight = tmp; - } - return info; - } - } - - @Override - public void process(final OperationList opList, - final Info imageInfo, - final OutputStream outputStream) throws FormatException, ProcessorException { - super.process(opList, imageInfo, outputStream); - - final ByteArrayOutputStream errorBucket = new ByteArrayOutputStream(); - - try { - if (isWindows()) { - processInWindows(opList, imageInfo, errorBucket, outputStream); - } else { - processInUnix(opList, imageInfo, errorBucket, outputStream); - } - } catch (EOFException e) { - // This is usually caused by the connection closing. - String msg = e.getMessage(); - msg = String.format("process(): %s (%s)", - (msg != null && msg.length() > 0) ? msg : "EOFException", - opList.toString()); - LOGGER.debug(msg, e); - throw new ProcessorException(msg, e); - } catch (IOException | InterruptedException e) { - final String errorStr = toString(errorBucket); - //if (errorStr.contains("does not contain a JPEG 2000 code stream")) { - throw new SourceFormatException(getSourceFormat()); - //} - //throw new ProcessorException( - // e.getMessage() + " (command output: " + errorStr + ")", e); - } - } - - private void processInWindows(final OperationList opList, - final Info info, - final ByteArrayOutputStream errorOutput, - final OutputStream outputStream) - throws IOException, InterruptedException { - // Will receive stdin output from grk_decompress (but none is expected). - final ByteArrayOutputStream inputBucket = new ByteArrayOutputStream(); - final Path intermediateFile = getIntermediateImageFile(opList); - final ReductionFactor reductionFactor = new ReductionFactor(); - final ThreadPool pool = ThreadPool.getInstance(); - - final ProcessBuilder pb = getProcessBuilder( - opList, info.getSize(), info.getNumResolutions(), - reductionFactor, intermediateFile); - LOGGER.debug("Invoking {}", String.join(" ", pb.command())); - final Process process = pb.start(); - - try (final InputStream processInputStream = - new BufferedInputStream(process.getInputStream()); - final InputStream processErrorStream = process.getErrorStream()) { - pool.submit(new StreamCopier(processErrorStream, errorOutput)); - pool.submit(new StreamCopier(processInputStream, inputBucket)); - - final int code = process.waitFor(); - if (code != 0) { - LOGGER.warn("grk_decompress returned with code {}", code); - String errorStr = toString(errorOutput); - errorStr += "\nPathname: " + getSourceFile(); - throw new IOException(errorStr); - } - } finally { - process.destroy(); - } - - try (InputStream is = Files.newInputStream(intermediateFile)) { - final ImageReader reader = - new ImageReaderFactory().newImageReader(Format.get("bmp"), is); - try { - final BufferedImage image = reader.read(0); - final Set hints = - EnumSet.of(ReaderHint.ALREADY_CROPPED); - - Java2DPostProcessor.postProcess( - image, hints, opList, info, reductionFactor); - - ImageWriterFacade.write(image, - (Encode) opList.getFirst(Encode.class), - outputStream); - } finally { - reader.dispose(); - } - } finally { - TaskQueue.getInstance().submit(() -> { - LOGGER.debug("Deleting {}", intermediateFile); - Files.delete(intermediateFile); - return null; - }); - } - } - - private void processInUnix(final OperationList opList, - final Info info, - final ByteArrayOutputStream errorOutput, - final OutputStream outputStream) - throws IOException, InterruptedException { - final ReductionFactor reductionFactor = new ReductionFactor(); - - final Path stdoutSymlink = createStdoutSymlink(); - final ProcessBuilder pb = getProcessBuilder( - opList, info.getSize(), info.getNumResolutions(), - reductionFactor, stdoutSymlink); - LOGGER.debug("Invoking {}", String.join(" ", pb.command())); - final Process process = pb.start(); - - try (final InputStream processInputStream = - new BufferedInputStream(process.getInputStream()); - final InputStream processErrorStream = process.getErrorStream()) { - ThreadPool.getInstance().submit( - new StreamCopier(processErrorStream, errorOutput)); - - final ImageReader reader = new ImageReaderFactory().newImageReader( - Format.get("bmp"), processInputStream); - try { - final Set hints = - EnumSet.of(ReaderHint.ALREADY_CROPPED); - - BufferedImage image = reader.read(0); - image = Java2DPostProcessor.postProcess( - image, hints, opList, info, reductionFactor); - - ImageWriterFacade.write(image, - (Encode) opList.getFirst(Encode.class), - outputStream); - - final int code = process.waitFor(); - if (code != 0) { - LOGGER.warn("{} returned with code {}", - GRK_DECOMPRESS_NAME, code); - String errorStr = toString(errorOutput); - errorStr += "\nPathname: " + getSourceFile(); - throw new IOException(errorStr); - } - } finally { - reader.dispose(); - } - } finally { - process.destroy(); - - TaskQueue.getInstance().submit(() -> { - LOGGER.debug("Deleting {}", stdoutSymlink); - Files.delete(stdoutSymlink); - return null; - }); - } - } - - /** - * Returns an instance corresponding to the given arguments. - * - * @param opList - * @param fullSize Full size of the source image. - * @param numResolutions Number of resolutions (DWT levels + 1) available - * in the source image. - * @param reduction The {@link ReductionFactor#factor} property will - * be modified. - * @param outputFile File to write to. - * @return {@link ProcessBuilder} for invoking {@literal - * grk_decompress} with arguments corresponding to - * the given arguments. - */ - private ProcessBuilder getProcessBuilder(final OperationList opList, - final Dimension fullSize, - final int numResolutions, - final ReductionFactor reduction, - final Path outputFile) { - final List command = new ArrayList<>(30); - command.add(getPath()); - - command.add("-i"); - command.add(getSourceFile().toString()); - - for (Operation op : opList) { - if (!op.hasEffect(fullSize, opList)) { - continue; - } - if (op instanceof Crop) { - final Crop crop = (Crop) op; - final Rectangle region = crop.getRectangle( - fullSize, opList.getScaleConstraint()); - command.add("-d"); - command.add(String.format("%d,%d,%d,%d", - region.intX(), region.intY(), - region.intX() + region.intWidth(), - region.intY() + region.intHeight())); - } else if (op instanceof Scale) { - // grk_decompress is not capable of arbitrary scaling, but it - // does offer a -r (reduce) argument to select a - // decomposition level, significantly speeding decompression. - // We can use it if the scale mode is ASPECT_FIT_* and either - // the percent is <=50, or the height/width are <=50% of full - // size. - final Scale scale = (Scale) op; - final Dimension tileSize = getROISize(opList, fullSize); - - int numDWTLevels = numResolutions - 1; - if (numDWTLevels < 0) { - numDWTLevels = FALLBACK_NUM_DWT_LEVELS; - } - reduction.factor = scale.getReductionFactor( - tileSize, opList.getScaleConstraint(), - numDWTLevels).factor; - - if (reduction.factor > 0) { - command.add("-r"); - command.add(reduction.factor + ""); - } - } - } - - command.add("-o"); - command.add(outputFile.toString()); - - return new ProcessBuilder(command); - } - - /** - * @return Size of the region of interest. - */ - private static Dimension getROISize(OperationList opList, - Dimension fullSize) { - Dimension size = new Dimension(fullSize); - for (Operation op : opList) { - if (op instanceof Crop) { - size = ((Crop) op).getRectangle( - size, opList.getScaleConstraint()).size(); - } - } - return size; - } - -} diff --git a/src/main/java/edu/illinois/library/cantaloupe/processor/ProcessorFactory.java b/src/main/java/edu/illinois/library/cantaloupe/processor/ProcessorFactory.java index fa7e18142..f2efc3287 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/processor/ProcessorFactory.java +++ b/src/main/java/edu/illinois/library/cantaloupe/processor/ProcessorFactory.java @@ -26,7 +26,6 @@ public final class ProcessorFactory { Java2dProcessor.class, KakaduNativeProcessor.class, OpenJpegProcessor.class, - GrokProcessor.class, PdfBoxProcessor.class, TurboJpegProcessor.class); diff --git a/src/main/resources/admin.vm b/src/main/resources/admin.vm index 8890f1648..a6cd4e714 100644 --- a/src/main/resources/admin.vm +++ b/src/main/resources/admin.vm @@ -2513,26 +2513,7 @@ - -
- - - - - -
Path to binaries - ? - - -
-
- +
diff --git a/src/test/java/edu/illinois/library/cantaloupe/perf/processor/GrokProcessorPerformance.java b/src/test/java/edu/illinois/library/cantaloupe/perf/processor/GrokProcessorPerformance.java deleted file mode 100644 index d1728d619..000000000 --- a/src/test/java/edu/illinois/library/cantaloupe/perf/processor/GrokProcessorPerformance.java +++ /dev/null @@ -1,72 +0,0 @@ -package edu.illinois.library.cantaloupe.perf.processor; - -import java.io.OutputStream; -import java.util.concurrent.TimeUnit; - -import edu.illinois.library.cantaloupe.config.Configuration; -import edu.illinois.library.cantaloupe.config.Key; -import edu.illinois.library.cantaloupe.image.Format; -import edu.illinois.library.cantaloupe.image.Info; -import edu.illinois.library.cantaloupe.operation.Encode; -import edu.illinois.library.cantaloupe.operation.OperationList; -import edu.illinois.library.cantaloupe.processor.FileProcessor; -import edu.illinois.library.cantaloupe.processor.ProcessorFactory; -import edu.illinois.library.cantaloupe.test.TestUtil; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.TearDown; -import org.openjdk.jmh.annotations.Warmup; - -import static edu.illinois.library.cantaloupe.test.PerformanceTestConstants.*; - -@BenchmarkMode(Mode.AverageTime) -@OutputTimeUnit(TimeUnit.MICROSECONDS) -@Warmup(iterations = WARMUP_ITERATIONS, - time = WARMUP_TIME) -@Measurement(iterations = MEASUREMENT_ITERATIONS, - time = MEASUREMENT_TIME) -@State(Scope.Benchmark) -@Fork(value = 1, jvmArgs = { "-server", "-Xms128M", "-Xmx128M", "-Dcantaloupe.config=memory" }) -public class GrokProcessorPerformance { - - private static final Format OUTPUT_FORMAT = Format.get("png"); - - private FileProcessor processor; - - @Setup - public void setUp() throws Exception { - Configuration config = Configuration.getInstance(); - config.setProperty(Key.PROCESSOR_FALLBACK, "GrokProcessor"); - processor = (FileProcessor) new ProcessorFactory().newProcessor(Format.get("jp2")); - } - - @TearDown - public void tearDown() { - processor.close(); - } - - @Benchmark - public void process() throws Exception { - processor.setSourceFormat(Format.get("jp2")); - processor.setSourceFile(TestUtil.getImage("jp2-5res-rgb-64x56x8-monotiled-lossy.jp2")); - processor.process( - OperationList.builder().withOperations(new Encode(OUTPUT_FORMAT)).build(), - Info.builder().withSize(64, 56).build(), - OutputStream.nullOutputStream()); - } - - @Benchmark - public void readInfo() throws Exception { - processor.setSourceFormat(Format.get("jp2")); - processor.setSourceFile(TestUtil.getImage("jp2-5res-rgb-64x56x8-monotiled-lossy.jp2")); - processor.readInfo(); - } - -} diff --git a/src/test/java/edu/illinois/library/cantaloupe/processor/AutomaticSelectionStrategyTest.java b/src/test/java/edu/illinois/library/cantaloupe/processor/AutomaticSelectionStrategyTest.java index f290124af..b30b42c1e 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/processor/AutomaticSelectionStrategyTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/processor/AutomaticSelectionStrategyTest.java @@ -23,8 +23,7 @@ public void setUp() throws Exception { void getPreferredProcessorsWithJP2() { List expected = List.of( KakaduNativeProcessor.class, - OpenJpegProcessor.class, - GrokProcessor.class); + OpenJpegProcessor.class); assertEquals(expected, instance.getPreferredProcessors(Format.get("jp2"))); } diff --git a/src/test/java/edu/illinois/library/cantaloupe/processor/GrokProcessorTest.java b/src/test/java/edu/illinois/library/cantaloupe/processor/GrokProcessorTest.java deleted file mode 100644 index 6948a8d47..000000000 --- a/src/test/java/edu/illinois/library/cantaloupe/processor/GrokProcessorTest.java +++ /dev/null @@ -1,111 +0,0 @@ -package edu.illinois.library.cantaloupe.processor; - -import edu.illinois.library.cantaloupe.config.Configuration; -import edu.illinois.library.cantaloupe.config.Key; -import edu.illinois.library.cantaloupe.image.Format; -import edu.illinois.library.cantaloupe.image.Info; -import edu.illinois.library.cantaloupe.test.TestUtil; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.*; - -public class GrokProcessorTest extends AbstractProcessorTest { - - private GrokProcessor instance; - - @BeforeEach - public void setUp() throws Exception { - super.setUp(); - - Configuration.getInstance().clearProperty( - Key.GROKPROCESSOR_PATH_TO_BINARIES); - GrokProcessor.resetInitialization(); - - instance = newInstance(); - } - - @AfterEach - public void tearDown() throws Exception { - super.tearDown(); - instance.close(); - } - - @Override - protected GrokProcessor newInstance() { - GrokProcessor proc = new GrokProcessor(); - try { - proc.setSourceFormat(Format.get("jp2")); - } catch (SourceFormatException e) { - fail("Huge bug"); - } - return proc; - } - - @Test - void testGetInitializationErrorWithNoException() { - assertNull(instance.getInitializationError()); - } - - @Test - void testGetInitializationErrorWithMissingBinaries() { - Configuration.getInstance().setProperty( - Key.GROKPROCESSOR_PATH_TO_BINARIES, - "/bogus/bogus/bogus"); - GrokProcessor.resetInitialization(); - assertNotNull(instance.getInitializationError()); - } - - @Test - void testReadInfoIPTCAwareness() throws Exception { - instance.setSourceFile(TestUtil.getImage("jp2-iptc.jp2")); - Info info = instance.readInfo(); - assertTrue(info.getMetadata().getIPTC().isPresent()); - } - - @Test - void testReadInfoXMPAwareness() throws Exception { - instance.setSourceFile(TestUtil.getImage("jp2-xmp.jp2")); - Info info = instance.readInfo(); - assertTrue(info.getMetadata().getXMP().isPresent()); - } - - @Test - void testReadInfoTileAwareness() throws Exception { - // untiled image - instance.setSourceFile(TestUtil.getImage("jp2-5res-rgb-64x56x8-monotiled-lossy.jp2")); - Info expectedInfo = Info.builder() - .withSize(64, 56) - .withTileSize(64, 56) - .withFormat(Format.get("jp2")) - .withNumResolutions(5) - .build(); - assertEquals(expectedInfo, instance.readInfo()); - - // tiled image - instance.setSourceFile(TestUtil.getImage("jp2-6res-rgb-64x56x8-multitiled-lossy.jp2")); - expectedInfo = Info.builder() - .withSize(64, 56) - .withTileSize(32, 28) - .withNumResolutions(6) - .withFormat(Format.get("jp2")) - .build(); - assertEquals(expectedInfo, instance.readInfo()); - } - - @Test - void testSupportsSourceFormatWithSupportedFormat() { - try (Processor instance = newInstance()) { - assertTrue(instance.supportsSourceFormat(Format.get("jp2"))); - } - } - - @Test - void testSupportsSourceFormatWithUnsupportedFormat() { - try (Processor instance = newInstance()) { - assertFalse(instance.supportsSourceFormat(Format.get("gif"))); - } - } - -} diff --git a/src/test/java/edu/illinois/library/cantaloupe/resource/admin/AdminResourceUITest.java b/src/test/java/edu/illinois/library/cantaloupe/resource/admin/AdminResourceUITest.java index a3092c0dd..b6a198a9c 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/resource/admin/AdminResourceUITest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/resource/admin/AdminResourceUITest.java @@ -525,9 +525,6 @@ void testProcessorsSection() throws Exception { // OpenJpegProcessor css("#cl-processors li > a[href=\"#OpenJpegProcessor\"]").click(); inputNamed(Key.OPENJPEGPROCESSOR_PATH_TO_BINARIES).sendKeys("/ojpath"); - // GrokProcessor - css("#cl-processors li > a[href=\"#GrokProcessor\"]").click(); - inputNamed(Key.GROKPROCESSOR_PATH_TO_BINARIES).sendKeys("/grkpath"); // PdfBoxProcessor css("#cl-processors li > a[href=\"#PdfBoxProcessor\"]").click(); inputNamed(Key.PROCESSOR_PDF_SCRATCH_FILE_ENABLED).click(); @@ -569,9 +566,6 @@ void testProcessorsSection() throws Exception { // OpenJpegProcessor assertEquals("/ojpath", config.getString(Key.OPENJPEGPROCESSOR_PATH_TO_BINARIES)); - // GrokProcessor - assertEquals("/grkpath", - config.getString(Key.GROKPROCESSOR_PATH_TO_BINARIES)); // PdfBoxProcessor assertEquals(-1, config.getLongBytes(Key.PROCESSOR_PDF_MAX_MEMORY_BYTES));