merge 2023.06.x into main

mycore-base/src/main/java/org/mycore/common/MCRDefaultHTTPClient.java

@@ -0,0 +1,119 @@
+/*
+ * This file is part of ***  M y C o R e  ***
+ * See http://www.mycore.de/ for details.
+ *
+ * MyCoRe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * MyCoRe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with MyCoRe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package org.mycore.common;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+
+import org.apache.http.client.cache.HttpCacheContext;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.cache.CacheConfig;
+import org.apache.http.impl.client.cache.CachingHttpClients;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.mycore.common.config.annotation.MCRProperty;
+import org.mycore.common.content.MCRContent;
+import org.mycore.common.content.MCRStreamContent;
+import org.mycore.common.events.MCRShutdownHandler;
+import org.mycore.services.http.MCRHttpUtils;
+
+public class MCRDefaultHTTPClient implements MCRHTTPClient {
+    private static Logger logger = LogManager.getLogger();
+
+    private long maxObjectSize;
+
+    private int maxCacheEntries;
+
+    private int requestTimeout;
+
+    private CloseableHttpClient restClient;
+
+    public MCRDefaultHTTPClient() {
+        CacheConfig cacheConfig = CacheConfig.custom()
+            .setMaxObjectSize(maxObjectSize)
+            .setMaxCacheEntries(maxCacheEntries)
+            .build();
+        RequestConfig requestConfig = RequestConfig.custom()
+            .setConnectTimeout(requestTimeout)
+            .setSocketTimeout(requestTimeout)
+            .build();
+        this.restClient = CachingHttpClients.custom()
+            .setCacheConfig(cacheConfig)
+            .setDefaultRequestConfig(requestConfig)
+            .setUserAgent(MCRHttpUtils.getHttpUserAgent())
+            .useSystemProperties()
+            .build();
+        MCRShutdownHandler.getInstance().addCloseable(this::close);
+    }
+
+    @MCRProperty(name = "MaxObjectSize")
+    public void setMaxObjectSize(String size) {
+        this.maxObjectSize = Long.parseLong(size);
+    }
+
+    @MCRProperty(name = "MaxCacheEntries")
+    public void setMaxCacheEntries(String size) {
+        this.maxCacheEntries = Integer.parseInt(size);
+    }
+
+    @MCRProperty(name = "RequestTimeout")
+    public void setRequestTimeout(String size) {
+        this.requestTimeout = Integer.parseInt(size);
+    }
+
+    public void close() {
+        try {
+            restClient.close();
+        } catch (IOException e) {
+            logger.warn("Exception while closing http client.", e);
+        }
+    }
+
+    @Override
+    public MCRContent get(URI hrefURI) throws IOException {
+        HttpCacheContext context = HttpCacheContext.create();
+        HttpGet get = new HttpGet(hrefURI);
+        MCRContent retContent = null;
+        try (CloseableHttpResponse response = restClient.execute(get, context);
+            InputStream content = response.getEntity().getContent();) {
+            logger.debug("http query: {}", hrefURI);
+            logger.debug("http resp status: {}", response.getStatusLine());
+            logger.debug(() -> getCacheDebugMsg(hrefURI, context));
+            retContent = (new MCRStreamContent(content)).getReusableCopy();
+        } finally {
+            get.reset();
+        }
+        return retContent;
+    }
+
+    private String getCacheDebugMsg(URI hrefURI, HttpCacheContext context) {
+        return hrefURI.toASCIIString() + ": " +
+            switch (context.getCacheResponseStatus()) {
+                case CACHE_HIT -> "A response was generated from the cache with no requests sent upstream";
+                case CACHE_MODULE_RESPONSE -> "The response was generated directly by the caching module";
+                case CACHE_MISS -> "The response came from an upstream server";
+                case VALIDATED -> "The response was generated from the cache after validating the entry "
+                    + "with the origin server";
+            };
+    }
+}

mycore-base/src/main/java/org/mycore/common/MCRHTTPClient.java

@@ -0,0 +1,30 @@
+/*
+ * This file is part of ***  M y C o R e  ***
+ * See http://www.mycore.de/ for details.
+ *
+ * MyCoRe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * MyCoRe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with MyCoRe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package org.mycore.common;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.mycore.common.content.MCRContent;
+
+public interface MCRHTTPClient {
+    MCRContent get(URI hrefURI) throws IOException;
+
+    void close();
+}

mycore-base/src/main/java/org/mycore/common/xml/MCRURIResolver.java

@@ -58,14 +58,7 @@
 import javax.xml.xpath.XPathExpressionException;
 
 import org.apache.commons.lang3.StringUtils;
-import org.apache.http.client.cache.HttpCacheContext;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.utils.URLEncodedUtils;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.cache.CacheConfig;
-import org.apache.http.impl.client.cache.CachingHttpClients;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.Marker;
@@ -82,6 +75,7 @@
 import org.mycore.common.MCRCoreVersion;
 import org.mycore.common.MCRDeveloperTools;
 import org.mycore.common.MCRException;
+import org.mycore.common.MCRHTTPClient;
 import org.mycore.common.MCRSessionMgr;
 import org.mycore.common.MCRUsageException;
 import org.mycore.common.MCRUserInformation;
@@ -95,7 +89,6 @@
 import org.mycore.common.content.transformer.MCRContentTransformer;
 import org.mycore.common.content.transformer.MCRParameterizedTransformer;
 import org.mycore.common.content.transformer.MCRXSLTransformer;
-import org.mycore.common.events.MCRShutdownHandler;
 import org.mycore.common.xsl.MCRLazyStreamSource;
 import org.mycore.common.xsl.MCRParameterCollector;
 import org.mycore.datamodel.classifications2.MCRCategory;
@@ -114,7 +107,6 @@
 import org.mycore.datamodel.niofs.MCRPath;
 import org.mycore.datamodel.niofs.MCRPathXML;
 import org.mycore.frontend.MCRLayoutUtilities;
-import org.mycore.services.http.MCRHttpUtils;
 import org.mycore.services.i18n.MCRTranslation;
 import org.mycore.tools.MCRObjectFactory;
 import org.xml.sax.InputSource;
@@ -140,6 +132,8 @@ public final class MCRURIResolver implements URIResolver {
 
     private static final String CONFIG_PREFIX = "MCR.URIResolver.";
 
+    private static final String HTTP_CLIENT_CLASS = "MCR.HTTPClient.Class";
+
     private static final Marker UNIQUE_MARKER = MarkerManager.getMarker("tryResolveXML");
 
     private static Map<String, URIResolver> SUPPORTED_SCHEMES;
@@ -539,77 +533,23 @@ public Source resolve(String href, String base) throws TransformerException {
     }
 
     private static class MCRRESTResolver implements URIResolver {
-        private static final long MAX_OBJECT_SIZE = MCRConfiguration2.getLong(CONFIG_PREFIX + "REST.MaxObjectSize")
-            .orElse(128 * 1024L);
-
-        private static final int MAX_CACHE_ENTRIES = MCRConfiguration2.getInt(CONFIG_PREFIX + "REST.MaxCacheEntries")
-            .orElse(1000);
-
-        private static final int REQUEST_TIMEOUT = MCRConfiguration2.getInt(CONFIG_PREFIX + "REST.RequestTimeout")
-            .orElse(30000);
-
-        private CloseableHttpClient restClient;
-
-        private org.apache.logging.log4j.Logger logger;
+        private MCRHTTPClient client;
 
         MCRRESTResolver() {
-            CacheConfig cacheConfig = CacheConfig.custom()
-                .setMaxObjectSize(MAX_OBJECT_SIZE)
-                .setMaxCacheEntries(MAX_CACHE_ENTRIES)
-                .build();
-            RequestConfig requestConfig = RequestConfig.custom()
-                .setConnectTimeout(REQUEST_TIMEOUT)
-                .setSocketTimeout(REQUEST_TIMEOUT)
-                .build();
-            this.restClient = CachingHttpClients.custom()
-                .setCacheConfig(cacheConfig)
-                .setDefaultRequestConfig(requestConfig)
-                .setUserAgent(MCRHttpUtils.getHttpUserAgent())
-                .useSystemProperties()
-                .build();
-            MCRShutdownHandler.getInstance().addCloseable(this::close);
-            this.logger = LogManager.getLogger();
-        }
-
-        public void close() {
-            try {
-                restClient.close();
-            } catch (IOException e) {
-                LogManager.getLogger().warn("Exception while closing http client.", e);
-            }
+            this.client = (MCRHTTPClient) MCRConfiguration2.getInstanceOf(HTTP_CLIENT_CLASS).get();
         }
 
         @Override
         public Source resolve(String href, String base) throws TransformerException {
             URI hrefURI = MCRURIResolver.resolveURI(href, base);
             try {
-                HttpCacheContext context = HttpCacheContext.create();
-                HttpGet get = new HttpGet(hrefURI);
-                try (CloseableHttpResponse response = restClient.execute(get, context);
-                    InputStream content = response.getEntity().getContent()) {
-                    logger.debug(() -> getCacheDebugMsg(hrefURI, context));
-                    final Source source = new MCRStreamContent(content).getReusableCopy().getSource();
-                    source.setSystemId(hrefURI.toASCIIString());
-                    return source;
-                } finally {
-                    get.reset();
-                }
+                final Source source = client.get(hrefURI).getSource();
+                source.setSystemId(hrefURI.toASCIIString());
+                return source;
             } catch (IOException e) {
                 throw new TransformerException(e);
             }
         }
-
-        private String getCacheDebugMsg(URI hrefURI, HttpCacheContext context) {
-            return hrefURI.toASCIIString() + ": " +
-                switch (context.getCacheResponseStatus()) {
-                    case CACHE_HIT -> "A response was generated from the cache with no requests sent upstream";
-                    case CACHE_MODULE_RESPONSE -> "The response was generated directly by the caching module";
-                    case CACHE_MISS -> "The response came from an upstream server";
-                    case VALIDATED -> "The response was generated from the cache after validating the entry "
-                        + "with the origin server";
-                };
-        }
-
     }
 
     private static class MCRObjectResolver implements URIResolver {

mycore-base/src/main/java/org/mycore/common/xml/MCRXMLFunctions.java

@@ -25,9 +25,11 @@
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.URLEncoder;
+import java.nio.file.FileStore;
 import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.text.Normalizer;
@@ -842,7 +844,7 @@ public static boolean isHtml(final String s) {
     }
 
     /**
-     * Strippes HTML tags from string.
+     * Stripes HTML tags from string.
      *
      * @param s string to strip HTML tags of
      * @return the plain text without tags
@@ -856,6 +858,20 @@ public static String stripHtml(final String s) {
         return StringEscapeUtils.unescapeHtml4(res.toString()).replaceAll(TAG_SELF_CLOSING, "");
     }
 
+    /**
+     * Returns approximately the usable space in the MCR.datadir in bytes as in {@link FileStore#getUsableSpace()}.
+     *
+     * @return approximately the usable space in the MCR.datadir
+     * @throws IOException
+     * */
+    public static long getUsableSpace() throws IOException{
+        Path dataDir = Paths.get(MCRConfiguration2.getStringOrThrow("MCR.datadir"));
+        dataDir = dataDir.toRealPath();
+        FileStore fs = Files.getFileStore(dataDir);
+
+        return fs.getUsableSpace();
+    }
+
     /**
      * Converts a string to valid NCName.
      *