Merge remote-tracking branch 'dspace/dspace-7_x' into backport-9019-t…

…o-dspace-7_x

docker-compose.yml

@@ -28,6 +28,7 @@ services:
       # proxies.trusted.ipranges: This setting is required for a REST API running in Docker to trust requests
       # from the host machine. This IP range MUST correspond to the 'dspacenet' subnet defined above.
       proxies__P__trusted__P__ipranges: '172.23.0'
+      LOGGING_CONFIG: /dspace/config/log4j2-container.xml
     image: "${DOCKER_OWNER:-dspace}/dspace:${DSPACE_VER:-dspace-7_x-test}"
     build:
       context: .

dspace-api/src/main/java/org/dspace/access/status/AccessStatusHelper.java

@@ -22,9 +22,21 @@ public interface AccessStatusHelper {
      *
      * @param context the DSpace context
      * @param item    the item
+     * @param threshold the embargo threshold date
      * @return an access status value
      * @throws SQLException An exception that provides information on a database access error or other errors.
      */
     public String getAccessStatusFromItem(Context context, Item item, Date threshold)
         throws SQLException;
+
+    /**
+     * Retrieve embargo information for the item
+     *
+     * @param context the DSpace context
+     * @param item the item to check for embargo information
+     * @param threshold the embargo threshold date
+     * @return an embargo date
+     * @throws SQLException An exception that provides information on a database access error or other errors.
+     */
+    public String getEmbargoFromItem(Context context, Item item, Date threshold) throws SQLException;
 }

dspace-api/src/main/java/org/dspace/access/status/AccessStatusServiceImpl.java

@@ -63,4 +63,9 @@ public void init() throws Exception {
     public String getAccessStatus(Context context, Item item) throws SQLException {
         return helper.getAccessStatusFromItem(context, item, forever_date);
     }
+
+    @Override
+    public String getEmbargoFromItem(Context context, Item item) throws SQLException {
+        return helper.getEmbargoFromItem(context, item, forever_date);
+    }
 }

dspace-api/src/main/java/org/dspace/access/status/DefaultAccessStatusHelper.java

@@ -26,13 +26,19 @@
 import org.dspace.core.Constants;
 import org.dspace.core.Context;
 import org.dspace.eperson.Group;
+import org.joda.time.LocalDate;
 
 /**
  * Default plugin implementation of the access status helper.
  * The getAccessStatusFromItem method provides a simple logic to
  * calculate the access status of an item based on the policies of
  * the primary or the first bitstream in the original bundle.
  * Users can override this method for enhanced functionality.
+ *
+ * The getEmbargoInformationFromItem method provides a simple logic to
+ *  * retrieve embargo information of bitstreams from an item based on the policies of
+ *  * the primary or the first bitstream in the original bundle.
+ *  * Users can override this method for enhanced functionality.
  */
 public class DefaultAccessStatusHelper implements AccessStatusHelper {
     public static final String EMBARGO = "embargo";
@@ -54,12 +60,12 @@ public DefaultAccessStatusHelper() {
 
     /**
      * Look at the item's policies to determine an access status value.
-     * It is also considering a date threshold for embargos and restrictions.
+     * It is also considering a date threshold for embargoes and restrictions.
      *
      * If the item is null, simply returns the "unknown" value.
      *
      * @param context     the DSpace context
-     * @param item        the item to embargo
+     * @param item        the item to check for embargoes
      * @param threshold   the embargo threshold date
      * @return an access status value
      */
@@ -86,7 +92,7 @@ public String getAccessStatusFromItem(Context context, Item item, Date threshold
                 .findFirst()
                 .orElse(null);
         }
-        return caculateAccessStatusForDso(context, bitstream, threshold);
+        return calculateAccessStatusForDso(context, bitstream, threshold);
     }
 
     /**
@@ -104,7 +110,7 @@ public String getAccessStatusFromItem(Context context, Item item, Date threshold
      * @param threshold   the embargo threshold date
      * @return an access status value
      */
-    private String caculateAccessStatusForDso(Context context, DSpaceObject dso, Date threshold)
+    private String calculateAccessStatusForDso(Context context, DSpaceObject dso, Date threshold)
             throws SQLException {
         if (dso == null) {
             return METADATA_ONLY;
@@ -156,4 +162,87 @@ private String caculateAccessStatusForDso(Context context, DSpaceObject dso, Dat
         }
         return RESTRICTED;
     }
+
+    /**
+     * Look at the policies of the primary (or first) bitstream of the item to retrieve its embargo.
+     *
+     * If the item is null, simply returns an empty map with no embargo information.
+     *
+     * @param context     the DSpace context
+     * @param item        the item to embargo
+     * @return an access status value
+     */
+    @Override
+    public String getEmbargoFromItem(Context context, Item item, Date threshold)
+            throws SQLException {
+        Date embargoDate;
+
+        // If Item status is not "embargo" then return a null embargo date.
+        String accessStatus = getAccessStatusFromItem(context, item, threshold);
+
+        if (item == null || !accessStatus.equals(EMBARGO)) {
+            return null;
+        }
+        // Consider only the original bundles.
+        List<Bundle> bundles = item.getBundles(Constants.DEFAULT_BUNDLE_NAME);
+        // Check for primary bitstreams first.
+        Bitstream bitstream = bundles.stream()
+                .map(bundle -> bundle.getPrimaryBitstream())
+                .filter(Objects::nonNull)
+                .findFirst()
+                .orElse(null);
+        if (bitstream == null) {
+            // If there is no primary bitstream,
+            // take the first bitstream in the bundles.
+            bitstream = bundles.stream()
+                    .map(bundle -> bundle.getBitstreams())
+                    .flatMap(List::stream)
+                    .findFirst()
+                    .orElse(null);
+        }
+
+        if (bitstream == null) {
+            return null;
+        }
+
+        embargoDate = this.retrieveShortestEmbargo(context, bitstream);
+
+        return embargoDate != null ? embargoDate.toString() : null;
+    }
+
+    /**
+     *
+     */
+    private Date retrieveShortestEmbargo(Context context, Bitstream bitstream) throws SQLException {
+        Date embargoDate = null;
+        // Only consider read policies.
+        List<ResourcePolicy> policies = authorizeService
+                .getPoliciesActionFilter(context, bitstream, Constants.READ);
+
+        // Looks at all read policies.
+        for (ResourcePolicy policy : policies) {
+            boolean isValid = resourcePolicyService.isDateValid(policy);
+            Group group = policy.getGroup();
+
+            if (group != null && StringUtils.equals(group.getName(), Group.ANONYMOUS)) {
+                // Only calculate the status for the anonymous group.
+                if (!isValid) {
+                    // If the policy is not valid there is an active embargo
+                    Date startDate = policy.getStartDate();
+
+                    if (startDate != null && !startDate.before(LocalDate.now().toDate())) {
+                        // There is an active embargo: aim to take the shortest embargo (account for rare cases where
+                        // more than one resource policy exists)
+                        if (embargoDate == null) {
+                            embargoDate = startDate;
+                        } else {
+                            embargoDate = startDate.before(embargoDate) ? startDate : embargoDate;
+                        }
+                    }
+                }
+            }
+        }
+
+        return embargoDate;
+    }
 }

dspace-api/src/main/java/org/dspace/access/status/service/AccessStatusService.java

@@ -40,7 +40,18 @@ public interface AccessStatusService {
      *
      * @param context the DSpace context
      * @param item    the item
+     * @return an access status value
      * @throws SQLException An exception that provides information on a database access error or other errors.
      */
     public String getAccessStatus(Context context, Item item) throws SQLException;
+
+    /**
+     * Retrieve embargo information for the item
+     *
+     * @param context the DSpace context
+     * @param item the item to check for embargo information
+     * @return an embargo date
+     * @throws SQLException An exception that provides information on a database access error or other errors.
+     */
+    public String getEmbargoFromItem(Context context, Item item) throws SQLException;
 }

dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java

@@ -578,6 +578,10 @@ public List<BulkEditChange> runImport(Context c, boolean change,
                             wfItem = workflowService.startWithoutNotify(c, wsItem);
                         }
                     } else {
+                        // Add provenance info
+                        String provenance = installItemService.getSubmittedByProvenanceMessage(c, wsItem.getItem());
+                        itemService.addMetadata(c, item, MetadataSchemaEnum.DC.getName(),
+                                "description", "provenance", "en", provenance);
                         // Install the item
                         installItemService.installItem(c, wsItem);
                     }
@@ -1363,7 +1367,7 @@ private int displayChanges(List<BulkEditChange> changes, boolean changed) {
      * is the field is defined as authority controlled
      */
     private static boolean isAuthorityControlledField(String md) {
-        String mdf = StringUtils.substringAfter(md, ":");
+        String mdf = md.contains(":") ? StringUtils.substringAfter(md, ":") : md;
         mdf = StringUtils.substringBefore(mdf, "[");
         return authorityControlled.contains(mdf);
     }

dspace-api/src/main/java/org/dspace/app/itemimport/ItemImportServiceImpl.java

@@ -774,6 +774,10 @@ protected Item addItem(Context c, List<Collection> mycollections, String path,
             // put item in system
             if (!isTest) {
                 try {
+                    // Add provenance info
+                    String provenance = installItemService.getSubmittedByProvenanceMessage(c, wi.getItem());
+                    itemService.addMetadata(c, wi.getItem(), MetadataSchemaEnum.DC.getName(),
+                        "description", "provenance", "en", provenance);
                     installItemService.installItem(c, wi, myhandle);
                 } catch (Exception e) {
                     workspaceItemService.deleteAll(c, wi);

dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java

@@ -189,7 +189,10 @@ public static void deleteSitemaps() throws IOException {
      */
     public static void generateSitemaps(boolean makeHTMLMap, boolean makeSitemapOrg) throws SQLException, IOException {
         String uiURLStem = configurationService.getProperty("dspace.ui.url");
-        String sitemapStem = uiURLStem + "/sitemap";
+        if (!uiURLStem.endsWith("/")) {
+            uiURLStem = uiURLStem + '/';
+        }
+        String sitemapStem = uiURLStem + "sitemap";
 
         File outputDir = new File(configurationService.getProperty("sitemap.dir"));
         if (!outputDir.exists() && !outputDir.mkdir()) {
@@ -212,7 +215,7 @@ public static void generateSitemaps(boolean makeHTMLMap, boolean makeSitemapOrg)
         List<Community> comms = communityService.findAll(c);
 
         for (Community comm : comms) {
-            String url = uiURLStem + "/communities/" + comm.getID();
+            String url = uiURLStem + "communities/" + comm.getID();
 
             if (makeHTMLMap) {
                 html.addURL(url, null);
@@ -227,7 +230,7 @@ public static void generateSitemaps(boolean makeHTMLMap, boolean makeSitemapOrg)
         List<Collection> colls = collectionService.findAll(c);
 
         for (Collection coll : colls) {
-            String url = uiURLStem + "/collections/" + coll.getID();
+            String url = uiURLStem + "collections/" + coll.getID();
 
             if (makeHTMLMap) {
                 html.addURL(url, null);
@@ -259,11 +262,11 @@ public static void generateSitemaps(boolean makeHTMLMap, boolean makeSitemapOrg)
                     && StringUtils.isNotBlank(discoverResult.getSearchDocument(
                         discoverResult.getIndexableObjects().get(0)).get(0).getSearchFieldValues("entityType").get(0))
                 ) {
-                    url = uiURLStem + "/entities/" + StringUtils.lowerCase(discoverResult.getSearchDocument(
+                    url = uiURLStem + "entities/" + StringUtils.lowerCase(discoverResult.getSearchDocument(
                             discoverResult.getIndexableObjects().get(0))
                         .get(0).getSearchFieldValues("entityType").get(0)) + "/" + i.getID();
                 } else {
-                    url = uiURLStem + "/items/" + i.getID();
+                    url = uiURLStem + "items/" + i.getID();
                 }
                 Date lastMod = i.getLastModified();
 

dspace-api/src/main/java/org/dspace/content/InstallItemServiceImpl.java

@@ -271,4 +271,28 @@ public String getBitstreamProvenanceMessage(Context context, Item myitem)
 
         return myMessage.toString();
     }
+
+    @Override
+    public String getSubmittedByProvenanceMessage(Context context, Item item) throws SQLException {
+        // get date
+        DCDate now = DCDate.getCurrent();
+
+        // Create provenance description
+        StringBuffer provmessage = new StringBuffer();
+
+        if (item.getSubmitter() != null) {
+            provmessage.append("Submitted by ").append(item.getSubmitter().getFullName())
+                .append(" (").append(item.getSubmitter().getEmail()).append(") on ")
+                .append(now.toString());
+        } else {
+            // else, null submitter
+            provmessage.append("Submitted by unknown (probably automated) on")
+                .append(now.toString());
+        }
+        provmessage.append("\n");
+
+        // add sizes and checksums of bitstreams
+        provmessage.append(getBitstreamProvenanceMessage(context, item));
+        return provmessage.toString();
+    }
 }

dspace-api/src/main/java/org/dspace/content/authority/ChoiceAuthorityServiceImpl.java

@@ -17,6 +17,7 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
 import org.dspace.app.util.DCInput;
@@ -557,6 +558,15 @@ public DSpaceControlledVocabularyIndex getVocabularyIndex(String nameVocab) {
             init();
             ChoiceAuthority source = this.getChoiceAuthorityByAuthorityName(nameVocab);
             if (source != null && source instanceof DSpaceControlledVocabulary) {
+
+                // First, check if this vocabulary index is disabled
+                String[] vocabulariesDisabled = configurationService
+                    .getArrayProperty("webui.browse.vocabularies.disabled");
+                if (vocabulariesDisabled != null && ArrayUtils.contains(vocabulariesDisabled, nameVocab)) {
+                    // Discard this vocabulary browse index
+                    return null;
+                }
+
                 Set<String> metadataFields = new HashSet<>();
                 Map<String, List<String>> formsToFields = this.authoritiesFormDefinitions.get(nameVocab);
                 for (Map.Entry<String, List<String>> formToField : formsToFields.entrySet()) {

dspace-api/src/main/java/org/dspace/content/service/InstallItemService.java

@@ -83,4 +83,15 @@ public Item restoreItem(Context c, InProgressSubmission is,
     public String getBitstreamProvenanceMessage(Context context, Item myitem)
         throws SQLException;
 
+    /**
+     * Generate provenance description of direct item submission (not through workflow).
+     *
+     * @param context context
+     * @param item  the item to generate description for
+     * @return provenance description
+     * @throws SQLException if database error
+     */
+    public String getSubmittedByProvenanceMessage(Context context, Item item)
+        throws SQLException;;
+
 }

dspace-api/src/main/java/org/dspace/google/GoogleAsyncEventListener.java

@@ -22,6 +22,8 @@
 import org.apache.commons.lang.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.dspace.content.Bitstream;
+import org.dspace.content.Bundle;
 import org.dspace.content.factory.ContentServiceFactory;
 import org.dspace.core.Constants;
 import org.dspace.core.Context;
@@ -77,7 +79,7 @@ public void receiveEvent(Event event) {
         UsageEvent usageEvent = (UsageEvent) event;
         LOGGER.debug("Usage event received " + event.getName());
 
-        if (isNotBitstreamViewEvent(usageEvent)) {
+        if (!isContentBitstream(usageEvent)) {
             return;
         }
 
@@ -171,9 +173,33 @@ private String getDocumentPath(HttpServletRequest request) {
         return documentPath;
     }
 
-    private boolean isNotBitstreamViewEvent(UsageEvent usageEvent) {
-        return usageEvent.getAction() != UsageEvent.Action.VIEW
-            || usageEvent.getObject().getType() != Constants.BITSTREAM;
+    /**
+     * Verifies if the usage event is a content bitstream view event, by checking if:<ul>
+     * <li>the usage event is a view event</li>
+     * <li>the object of the usage event is a bitstream</li>
+     * <li>the bitstream belongs to one of the configured bundles (fallback: ORIGINAL bundle)</li></ul>
+     */
+    private boolean isContentBitstream(UsageEvent usageEvent) {
+        // check if event is a VIEW event and object is a Bitstream
+        if (usageEvent.getAction() == UsageEvent.Action.VIEW
+            && usageEvent.getObject().getType() == Constants.BITSTREAM) {
+            // check if bitstream belongs to a configured bundle
+            List<String> allowedBundles = List.of(configurationService
+                      .getArrayProperty("google-analytics.bundles", new String[]{Constants.CONTENT_BUNDLE_NAME}));
+            if (allowedBundles.contains("none")) {
+                // GA events for bitstream views were turned off in config
+                return false;
+            }
+            List<String> bitstreamBundles;
+            try {
+                bitstreamBundles = ((Bitstream) usageEvent.getObject())
+                    .getBundles().stream().map(Bundle::getName).collect(Collectors.toList());
+            } catch (SQLException e) {
+                throw new RuntimeException(e.getMessage(), e);
+            }
+            return allowedBundles.stream().anyMatch(bitstreamBundles::contains);
+        }
+        return false;
     }
 
     private boolean isGoogleAnalyticsKeyNotConfigured() {