Merge remote-tracking branch 'origin/dspace-7_x' into w2p-121973_open…

…aire-compliance-updates-7.x

Dockerfile

@@ -42,14 +42,12 @@ WORKDIR /dspace-src
 ENV ANT_VERSION=1.10.13
 ENV ANT_HOME=/tmp/ant-$ANT_VERSION
 ENV PATH=$ANT_HOME/bin:$PATH
-# Need wget to install ant
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends wget \
-    && apt-get purge -y --auto-remove \
-    && rm -rf /var/lib/apt/lists/*
 # Download and install 'ant'
 RUN mkdir $ANT_HOME && \
-    wget -qO- "https://archive.apache.org/dist/ant/binaries/apache-ant-$ANT_VERSION-bin.tar.gz" | tar -zx --strip-components=1 -C $ANT_HOME
+    curl --silent --show-error --location --fail --retry 5 --output /tmp/apache-ant.tar.gz \
+      https://archive.apache.org/dist/ant/binaries/apache-ant-${ANT_VERSION}-bin.tar.gz && \
+    tar -zx --strip-components=1 -f /tmp/apache-ant.tar.gz -C $ANT_HOME && \
+    rm /tmp/apache-ant.tar.gz
 # Run necessary 'ant' deploy scripts
 RUN ant init_installation update_configs update_code update_webapps
 

Dockerfile.cli

@@ -38,14 +38,12 @@ WORKDIR /dspace-src
 ENV ANT_VERSION=1.10.13
 ENV ANT_HOME=/tmp/ant-$ANT_VERSION
 ENV PATH=$ANT_HOME/bin:$PATH
-# Need wget to install ant
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends wget \
-    && apt-get purge -y --auto-remove \
-    && rm -rf /var/lib/apt/lists/*
 # Download and install 'ant'
 RUN mkdir $ANT_HOME && \
-    wget -qO- "https://archive.apache.org/dist/ant/binaries/apache-ant-$ANT_VERSION-bin.tar.gz" | tar -zx --strip-components=1 -C $ANT_HOME
+    curl --silent --show-error --location --fail --retry 5 --output /tmp/apache-ant.tar.gz \
+      https://archive.apache.org/dist/ant/binaries/apache-ant-${ANT_VERSION}-bin.tar.gz && \
+    tar -zx --strip-components=1 -f /tmp/apache-ant.tar.gz -C $ANT_HOME && \
+    rm /tmp/apache-ant.tar.gz
 # Run necessary 'ant' deploy scripts
 RUN ant init_installation update_configs update_code
 

Dockerfile.dependencies

@@ -6,7 +6,7 @@
 # To build with JDK17, use "--build-arg JDK_VERSION=17"
 ARG JDK_VERSION=11
 
-# Step 1 - Run Maven Build
+# Step 1 - Download all Dependencies
 FROM docker.io/maven:3-eclipse-temurin-${JDK_VERSION} AS build
 ARG TARGET_DIR=dspace-installer
 WORKDIR /app
@@ -19,16 +19,60 @@ RUN chown -Rv dspace: /app
 # Switch to dspace user & run below commands as that user
 USER dspace
 
-# Copy the DSpace source code (from local machine) into the workdir (excluding .dockerignore contents)
-ADD --chown=dspace . /app/
+# This next part may look odd, but it speeds up the build of this image *significantly*.
+# Copy ONLY the POMs to this image (from local machine). This will allow us to download all dependencies *without*
+# performing any code compilation steps.
+
+# Parent POM
+ADD --chown=dspace pom.xml /app/
+RUN mkdir -p /app/dspace
+
+# 'dspace' module POM. Includes 'additions' ONLY, as it's the only submodule that is required to exist.
+ADD --chown=dspace dspace/pom.xml /app/dspace/
+RUN mkdir -p /app/dspace/modules/
+ADD --chown=dspace dspace/modules/pom.xml /app/dspace/modules/
+RUN mkdir -p /app/dspace/modules/additions
+ADD --chown=dspace dspace/modules/additions/pom.xml /app/dspace/modules/additions/
+
+# 'dspace-api' module POM
+RUN mkdir -p /app/dspace-api
+ADD --chown=dspace dspace-api/pom.xml /app/dspace-api/
+
+# 'dspace-iiif' module POM
+RUN mkdir -p /app/dspace-iiif
+ADD --chown=dspace dspace-iiif/pom.xml /app/dspace-iiif/
+
+# 'dspace-oai' module POM
+RUN mkdir -p /app/dspace-oai
+ADD --chown=dspace dspace-oai/pom.xml /app/dspace-oai/
+
+# 'dspace-rdf' module POM
+RUN mkdir -p /app/dspace-rdf
+ADD --chown=dspace dspace-rdf/pom.xml /app/dspace-rdf/
+
+# 'dspace-server-webapp' module POM
+RUN mkdir -p /app/dspace-server-webapp
+ADD --chown=dspace dspace-server-webapp/pom.xml /app/dspace-server-webapp/
+
+# 'dspace-services' module POM
+RUN mkdir -p /app/dspace-services
+ADD --chown=dspace dspace-services/pom.xml /app/dspace-services/
+
+# 'dspace-sword' module POM
+RUN mkdir -p /app/dspace-sword
+ADD --chown=dspace dspace-sword/pom.xml /app/dspace-sword/
+
+# 'dspace-swordv2' module POM
+RUN mkdir -p /app/dspace-swordv2
+ADD --chown=dspace dspace-swordv2/pom.xml /app/dspace-swordv2/
 
 # Trigger the installation of all maven dependencies (hide download progress messages)
 # Maven flags here ensure that we skip final assembly, skip building test environment and skip all code verification checks.
-# These flags speed up this installation as much as reasonably possible.
-ENV MAVEN_FLAGS="-P-assembly -P-test-environment -Denforcer.skip=true -Dcheckstyle.skip=true -Dlicense.skip=true -Dxml.skip=true"
-RUN mvn --no-transfer-progress install ${MAVEN_FLAGS}
+# These flags speed up this installation and skip tasks we cannot perform as we don't have the full source code.
+ENV MAVEN_FLAGS="-P-assembly -P-test-environment -Denforcer.skip=true -Dcheckstyle.skip=true -Dlicense.skip=true -Dxjc.skip=true -Dxml.skip=true"
+RUN mvn --no-transfer-progress verify ${MAVEN_FLAGS}
 
-# Clear the contents of the /app directory (including all maven builds), so no artifacts remain.
+# Clear the contents of the /app directory (including all maven target folders), so no artifacts remain.
 # This ensures when dspace:dspace is built, it will use the Maven local cache (~/.m2) for dependencies
 USER root
 RUN rm -rf /app/*

Dockerfile.test

@@ -41,14 +41,12 @@ WORKDIR /dspace-src
 ENV ANT_VERSION=1.10.12
 ENV ANT_HOME=/tmp/ant-$ANT_VERSION
 ENV PATH=$ANT_HOME/bin:$PATH
-# Need wget to install ant
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends wget \
-    && apt-get purge -y --auto-remove \
-    && rm -rf /var/lib/apt/lists/*
 # Download and install 'ant'
 RUN mkdir $ANT_HOME && \
-    wget -qO- "https://archive.apache.org/dist/ant/binaries/apache-ant-$ANT_VERSION-bin.tar.gz" | tar -zx --strip-components=1 -C $ANT_HOME
+    curl --silent --show-error --location --fail --retry 5 --output /tmp/apache-ant.tar.gz \
+      https://archive.apache.org/dist/ant/binaries/apache-ant-${ANT_VERSION}-bin.tar.gz && \
+    tar -zx --strip-components=1 -f /tmp/apache-ant.tar.gz -C $ANT_HOME && \
+    rm /tmp/apache-ant.tar.gz
 # Run necessary 'ant' deploy scripts
 RUN ant init_installation update_configs update_code update_webapps
 

dspace-api/src/main/java/org/dspace/app/statistics/LogAnalyser.java

@@ -281,10 +281,14 @@ public class LogAnalyser {
      */
     private static String fileTemplate = "dspace\\.log.*";
 
+    private static final ConfigurationService configurationService =
+            DSpaceServicesFactory.getInstance().getConfigurationService();
+
     /**
      * the configuration file from which to configure the analyser
      */
-    private static String configFile;
+    private static String configFile = configurationService.getProperty("dspace.dir")
+            + File.separator + "config" + File.separator + "dstat.cfg";
 
     /**
      * the output file to which to write aggregation data
@@ -616,8 +620,6 @@ public static String processLogs(Context context, String myLogDir,
         }
 
         // now do the host name and url lookup
-        ConfigurationService configurationService
-                = DSpaceServicesFactory.getInstance().getConfigurationService();
         hostName = Utils.getHostName(configurationService.getProperty("dspace.ui.url"));
         name = configurationService.getProperty("dspace.name").trim();
         url = configurationService.getProperty("dspace.ui.url").trim();
@@ -658,8 +660,6 @@ public static void setParameters(String myLogDir, String myFileTemplate,
                                      String myConfigFile, String myOutFile,
                                      Date myStartDate, Date myEndDate,
                                      boolean myLookUp) {
-        ConfigurationService configurationService
-                = DSpaceServicesFactory.getInstance().getConfigurationService();
 
         if (myLogDir != null) {
             logDir = myLogDir;
@@ -673,9 +673,6 @@ public static void setParameters(String myLogDir, String myFileTemplate,
 
         if (myConfigFile != null) {
             configFile = myConfigFile;
-        } else {
-            configFile = configurationService.getProperty("dspace.dir")
-                    + File.separator + "config" + File.separator + "dstat.cfg";
         }
 
         if (myStartDate != null) {

dspace-api/src/main/java/org/dspace/app/util/DCInput.java

@@ -163,7 +163,7 @@ public class DCInput {
      * The scope of the input sets, this restricts hidden metadata fields from
      * view by the end user during submission.
      */
-    public static final String SUBMISSION_SCOPE = "submit";
+    public static final String SUBMISSION_SCOPE = "submission";
 
     /**
      * Class constructor for creating a DCInput object based on the contents of
@@ -262,7 +262,7 @@ protected void initRegex(String regex) {
 
     /**
      * Is this DCInput for display in the given scope? The scope should be
-     * either "workflow" or "submit", as per the input forms definition. If the
+     * either "workflow" or "submission", as per the input forms definition. If the
      * internal visibility is set to "null" then this will always return true.
      *
      * @param scope String identifying the scope that this input's visibility

dspace-api/src/main/java/org/dspace/browse/BrowseEngine.java

@@ -422,9 +422,6 @@ private BrowseInfo browseByValue(BrowserScope bs)
                 }
             }
 
-            // this is the total number of results in answer to the query
-            int total = getTotalResults(true);
-
             // set the ordering field (there is only one option)
             dao.setOrderField("sort_value");
 
@@ -444,6 +441,9 @@ private BrowseInfo browseByValue(BrowserScope bs)
             dao.setOffset(offset);
             dao.setLimit(scope.getResultsPerPage());
 
+            // this is the total number of results in answer to the query
+            int total = getTotalResults(true);
+
             // Holder for the results
             List<String[]> results = null;
 
@@ -680,33 +680,9 @@ private int getTotalResults(boolean distinct)
         // tell the browse query whether we are distinct
         dao.setDistinct(distinct);
 
-        // ensure that the select is set to "*"
-        String[] select = {"*"};
-        dao.setCountValues(select);
-
-        // FIXME: it would be nice to have a good way of doing this in the DAO
-        // now reset all of the fields that we don't want to have constraining
-        // our count, storing them locally to reinstate later
-        String focusField = dao.getJumpToField();
-        String focusValue = dao.getJumpToValue();
-        int limit = dao.getLimit();
-        int offset = dao.getOffset();
-
-        dao.setJumpToField(null);
-        dao.setJumpToValue(null);
-        dao.setLimit(-1);
-        dao.setOffset(-1);
-
         // perform the query and get the result
         int count = dao.doCountQuery();
 
-        // now put back the values we removed for this method
-        dao.setJumpToField(focusField);
-        dao.setJumpToValue(focusValue);
-        dao.setLimit(limit);
-        dao.setOffset(offset);
-        dao.setCountValues(null);
-
         log.debug(LogHelper.getHeader(context, "get_total_results_return", "return=" + count));
 
         return count;

dspace-api/src/main/java/org/dspace/browse/SolrBrowseDAO.java

@@ -13,6 +13,8 @@
 import java.util.Comparator;
 import java.util.List;
 
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
 import org.apache.solr.client.solrj.util.ClientUtils;
@@ -180,18 +182,33 @@ private DiscoverResult getSolrResponse() throws BrowseException {
             addDefaultFilterQueries(query);
             if (distinct) {
                 DiscoverFacetField dff;
+
+                // To get the number of distinct values we use the next "json.facet" query param
+                // {"entries_count": {"type":"terms","field": "<fieldName>_filter", "limit":0, "numBuckets":true}}"
+                ObjectNode jsonFacet = JsonNodeFactory.instance.objectNode();
+                ObjectNode entriesCount = JsonNodeFactory.instance.objectNode();
+                entriesCount.put("type", "terms");
+                entriesCount.put("field", facetField + "_filter");
+                entriesCount.put("limit", 0);
+                entriesCount.put("numBuckets", true);
+                jsonFacet.set("entries_count", entriesCount);
+
                 if (StringUtils.isNotBlank(startsWith)) {
                     dff = new DiscoverFacetField(facetField,
-                        DiscoveryConfigurationParameters.TYPE_TEXT, -1,
-                        DiscoveryConfigurationParameters.SORT.VALUE, startsWith);
+                        DiscoveryConfigurationParameters.TYPE_TEXT, limit,
+                        DiscoveryConfigurationParameters.SORT.VALUE, startsWith, offset);
+
+                    // Add the prefix to the json facet query
+                    entriesCount.put("prefix", startsWith);
                 } else {
                     dff = new DiscoverFacetField(facetField,
-                        DiscoveryConfigurationParameters.TYPE_TEXT, -1,
-                        DiscoveryConfigurationParameters.SORT.VALUE);
+                        DiscoveryConfigurationParameters.TYPE_TEXT, limit,
+                        DiscoveryConfigurationParameters.SORT.VALUE, offset);
                 }
                 query.addFacetField(dff);
                 query.setFacetMinCount(1);
                 query.setMaxResults(0);
+                query.addProperty("json.facet", jsonFacet.toString());
             } else {
                 query.setMaxResults(limit/* > 0 ? limit : 20*/);
                 if (offset > 0) {
@@ -248,8 +265,7 @@ public int doCountQuery() throws BrowseException {
         DiscoverResult resp = getSolrResponse();
         int count = 0;
         if (distinct) {
-            List<FacetResult> facetResults = resp.getFacetResult(facetField);
-            count = facetResults.size();
+            count = (int) resp.getTotalEntries();
         } else {
             // we need to cast to int to respect the BrowseDAO contract...
             count = (int) resp.getTotalSearchResults();
@@ -266,8 +282,8 @@ public List doValueQuery() throws BrowseException {
         DiscoverResult resp = getSolrResponse();
         List<FacetResult> facet = resp.getFacetResult(facetField);
         int count = doCountQuery();
-        int start = offset > 0 ? offset : 0;
-        int max = limit > 0 ? limit : count; //if negative, return everything
+        int start = 0;
+        int max = facet.size();
         List<String[]> result = new ArrayList<>();
         if (ascending) {
             for (int i = start; i < (start + max) && i < count; i++) {

dspace-api/src/main/java/org/dspace/core/Context.java

@@ -883,7 +883,19 @@ public <E extends ReloadableEntity> E reloadEntity(E entity) throws SQLException
     }
 
     /**
-     * Remove an entity from the cache. This is necessary when batch processing a large number of items.
+     * Remove all entities from the cache and reload the current user entity. This is useful when batch processing
+     * a large number of entities when the calling code requires the cache to be completely cleared before continuing.
+     *
+     * @throws SQLException if a database error occurs.
+     */
+    public void uncacheEntities() throws SQLException {
+        dbConnection.uncacheEntities();
+        reloadContextBoundEntities();
+    }
+
+    /**
+     * Remove an entity from the cache. This is useful when batch processing a large number of entities
+     * when the calling code needs to retain some items in the cache while removing others.
      *
      * @param entity The entity to reload
      * @param <E>    The class of the entity. The entity must implement the {@link ReloadableEntity} interface.

dspace-api/src/main/java/org/dspace/core/DBConnection.java

@@ -124,28 +124,38 @@ public interface DBConnection<T> {
     public long getCacheSize() throws SQLException;
 
     /**
-     * Reload a DSpace object from the database. This will make sure the object
+     * Reload an entity from the database. This will make sure the object
      * is valid and stored in the cache.  The returned object should be used
      * henceforth instead of the passed object.
      *
-     * @param <E>    type of {@link entity}
-     * @param entity The DSpace object to reload
+     * @param <E>    type of entity.
+     * @param entity The entity to reload.
      * @return the reloaded entity.
-     * @throws java.sql.SQLException passed through.
+     * @throws SQLException passed through.
      */
     public <E extends ReloadableEntity> E reloadEntity(E entity) throws SQLException;
 
     /**
-     * Remove a DSpace object from the session cache when batch processing a
-     * large number of objects.
+     * Remove all entities from the session cache.
      *
-     * <p>Objects removed from cache are not saved in any way. Therefore, if you
-     * have modified an object, you should be sure to {@link commit()} changes
+     * <p>Entities removed from cache are not saved in any way. Therefore, if you
+     * have modified any entities, you should be sure to {@link #commit()} changes
      * before calling this method.
      *
-     * @param <E>    Type of {@link entity}
-     * @param entity The DSpace object to decache.
-     * @throws java.sql.SQLException passed through.
+     * @throws SQLException passed through.
+     */
+    public void uncacheEntities() throws SQLException;
+
+    /**
+     * Remove an entity from the session cache.
+     *
+     * <p>Entities removed from cache are not saved in any way. Therefore, if you
+     * have modified the entity, you should be sure to {@link #commit()} changes
+     * before calling this method.
+     *
+     * @param <E>    Type of entity.
+     * @param entity The entity to decache.
+     * @throws SQLException passed through.
      */
     public <E extends ReloadableEntity> void uncacheEntity(E entity) throws SQLException;
 

dspace-api/src/main/java/org/dspace/core/HibernateDBConnection.java

@@ -243,6 +243,11 @@ private void configureDatabaseMode() throws SQLException {
         }
     }
 
+    @Override
+    public void uncacheEntities() throws SQLException {
+        getSession().clear();
+    }
+
     /**
      * Evict an entity from the hibernate cache.
      * <P>